@sproutboat/wire 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/package.json +1 -1
- package/src/broker.test.ts +301 -0
- package/src/broker.ts +798 -49
package/src/broker.ts
CHANGED
|
@@ -18,8 +18,9 @@
|
|
|
18
18
|
* the only SSRF control today.
|
|
19
19
|
*/
|
|
20
20
|
import { Database, type Statement } from "bun:sqlite";
|
|
21
|
-
import { createHash } from "node:crypto";
|
|
22
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
21
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
22
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statfsSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { open } from "node:fs/promises";
|
|
23
24
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
24
25
|
import { parseArgs } from "node:util";
|
|
25
26
|
import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
|
|
@@ -84,6 +85,16 @@ export type BrokerOptions = {
|
|
|
84
85
|
* data outlives a redeploy.
|
|
85
86
|
*/
|
|
86
87
|
resourceDir?: string;
|
|
88
|
+
/** Account whose R2 resources this broker may mutate. */
|
|
89
|
+
ownerId?: string;
|
|
90
|
+
/** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
|
|
91
|
+
r2QuotaBytes?: number;
|
|
92
|
+
/** Resource ids belonging to ownerId, used to seed the shared R2 usage ledger. */
|
|
93
|
+
r2ResourceIds?: string[];
|
|
94
|
+
/** Free disk capacity retained while direct R2 uploads are accepted. */
|
|
95
|
+
r2MinFreeBytes?: number;
|
|
96
|
+
/** Test hook for checking the filesystem capacity available to temporary uploads. */
|
|
97
|
+
freeBytes?: (path: string) => number;
|
|
87
98
|
token?: string;
|
|
88
99
|
bindings?: Partial<Bindings>;
|
|
89
100
|
secrets?: Record<string, string>;
|
|
@@ -115,7 +126,28 @@ type R2Row = {
|
|
|
115
126
|
uploaded: string;
|
|
116
127
|
http_json: string;
|
|
117
128
|
custom_json: string;
|
|
129
|
+
blob_id: string | null;
|
|
118
130
|
};
|
|
131
|
+
type R2UploadRow = {
|
|
132
|
+
key: string;
|
|
133
|
+
uploaded: string;
|
|
134
|
+
http_json: string;
|
|
135
|
+
custom_json: string;
|
|
136
|
+
};
|
|
137
|
+
type R2PartRow = { part_number: number; size: number; etag: string };
|
|
138
|
+
type R2UploadIdentity = { bucket: string; key: string; upload_id: string };
|
|
139
|
+
type R2TransferRow = {
|
|
140
|
+
bucket: string;
|
|
141
|
+
key: string;
|
|
142
|
+
method: string;
|
|
143
|
+
expires: number;
|
|
144
|
+
max_bytes: number;
|
|
145
|
+
expected_sha256: string;
|
|
146
|
+
http_json: string;
|
|
147
|
+
custom_json: string;
|
|
148
|
+
used: number;
|
|
149
|
+
};
|
|
150
|
+
const MULTIPART_EXPIRY_MS = 7 * 24 * 60 * 60 * 1000;
|
|
119
151
|
const isNumber = (v: JsonValue | undefined): v is number => Number.isFinite(v);
|
|
120
152
|
const sqlParams = (v: JsonValue | undefined): SqlParam[] => {
|
|
121
153
|
if (!Array.isArray(v)) return [];
|
|
@@ -144,6 +176,8 @@ export type Broker = {
|
|
|
144
176
|
* negotiation.
|
|
145
177
|
*/
|
|
146
178
|
handleFrame(payload: Buffer): Promise<Buffer>;
|
|
179
|
+
/** Serve one reserved direct R2 upload or download request. */
|
|
180
|
+
transfer(request: Request): Promise<Response>;
|
|
147
181
|
close(): void;
|
|
148
182
|
};
|
|
149
183
|
|
|
@@ -241,8 +275,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
241
275
|
conn.exec(
|
|
242
276
|
"CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
|
|
243
277
|
"etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
278
|
+
"blob_id TEXT, " +
|
|
244
279
|
"PRIMARY KEY (bucket, key))",
|
|
245
280
|
);
|
|
281
|
+
const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
|
|
282
|
+
if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
|
|
283
|
+
conn.exec(
|
|
284
|
+
"CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
285
|
+
"uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
286
|
+
"PRIMARY KEY (bucket, key, upload_id))",
|
|
287
|
+
);
|
|
288
|
+
conn.exec(
|
|
289
|
+
"CREATE TABLE IF NOT EXISTS r2_part (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
290
|
+
"part_number INTEGER NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, " +
|
|
291
|
+
"PRIMARY KEY (bucket, key, upload_id, part_number))",
|
|
292
|
+
);
|
|
293
|
+
conn.exec(
|
|
294
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer (token TEXT PRIMARY KEY, bucket TEXT NOT NULL, key TEXT NOT NULL, " +
|
|
295
|
+
"method TEXT NOT NULL, expires INTEGER NOT NULL, max_bytes INTEGER NOT NULL, expected_sha256 TEXT NOT NULL DEFAULT '', " +
|
|
296
|
+
"http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', used INTEGER NOT NULL DEFAULT 0)",
|
|
297
|
+
);
|
|
246
298
|
conn.exec(
|
|
247
299
|
"CREATE TABLE IF NOT EXISTS mq (queue TEXT NOT NULL, id TEXT PRIMARY KEY, body TEXT NOT NULL, " +
|
|
248
300
|
"visible_at INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, dead INTEGER NOT NULL DEFAULT 0)",
|
|
@@ -290,6 +342,121 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
290
342
|
return conn;
|
|
291
343
|
};
|
|
292
344
|
|
|
345
|
+
const quotaOwner = opts.ownerId;
|
|
346
|
+
const quotaBytes = opts.r2QuotaBytes && opts.r2QuotaBytes > 0 ? opts.r2QuotaBytes : null;
|
|
347
|
+
const quotaDirectory = resourceDir;
|
|
348
|
+
if (quotaOwner && quotaDirectory && quotaBytes) mkdirSync(quotaDirectory, { recursive: true });
|
|
349
|
+
// One ledger for every account, shared by the brokers of all its deployments.
|
|
350
|
+
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
351
|
+
// requests cannot each independently decide that the same remaining bytes fit.
|
|
352
|
+
const quotaDb = quotaOwner && quotaDirectory && quotaBytes
|
|
353
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
354
|
+
: null;
|
|
355
|
+
if (quotaDb) {
|
|
356
|
+
const owner = quotaOwner!;
|
|
357
|
+
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
358
|
+
quotaDb.exec(
|
|
359
|
+
"CREATE TABLE IF NOT EXISTS r2_account_quota (owner TEXT PRIMARY KEY, used_bytes INTEGER NOT NULL, reserved_bytes INTEGER NOT NULL)",
|
|
360
|
+
);
|
|
361
|
+
quotaDb.exec(
|
|
362
|
+
"CREATE TABLE IF NOT EXISTS r2_quota_reservation (token TEXT PRIMARY KEY, owner TEXT NOT NULL, bytes INTEGER NOT NULL, expires INTEGER NOT NULL)",
|
|
363
|
+
);
|
|
364
|
+
quotaDb.exec(
|
|
365
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
366
|
+
);
|
|
367
|
+
const existing = quotaDb.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?").get(owner);
|
|
368
|
+
if (!existing) {
|
|
369
|
+
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
370
|
+
}
|
|
371
|
+
quotaDb.query("INSERT OR IGNORE INTO r2_transfer_metric (owner) VALUES (?)").run(owner);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
const committedR2Bytes = (): number => {
|
|
375
|
+
if (!quotaDirectory) return 0;
|
|
376
|
+
let used = 0;
|
|
377
|
+
for (const id of opts.r2ResourceIds ?? []) {
|
|
378
|
+
// Open through resourceDb so older resource files gain the multipart
|
|
379
|
+
// tables before the aggregate is queried.
|
|
380
|
+
const store = resourceDb(id);
|
|
381
|
+
used += store.query<{ bytes: number }, []>(
|
|
382
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
383
|
+
).get()?.bytes ?? 0;
|
|
384
|
+
}
|
|
385
|
+
return used;
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
const availableBytes = (): number => {
|
|
389
|
+
if (!quotaDirectory) return Number.POSITIVE_INFINITY;
|
|
390
|
+
if (opts.freeBytes) return opts.freeBytes(quotaDirectory);
|
|
391
|
+
const stats = statfsSync(quotaDirectory);
|
|
392
|
+
return Number(stats.bavail) * Number(stats.bsize);
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
396
|
+
if (!quotaDb || !quotaOwner) return;
|
|
397
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
398
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
399
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
400
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
401
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
const reserveR2 = (token: string, bytes: number, expires: number): "ok" | "quota" | "disk" => {
|
|
405
|
+
if (!quotaDb || !quotaOwner || !quotaBytes) return "ok";
|
|
406
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
407
|
+
try {
|
|
408
|
+
releaseExpiredR2Reservations();
|
|
409
|
+
const used = committedR2Bytes();
|
|
410
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
411
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
412
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
413
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
414
|
+
if (used + reserved + bytes > quotaBytes) {
|
|
415
|
+
quotaDb.exec("ROLLBACK");
|
|
416
|
+
return "quota";
|
|
417
|
+
}
|
|
418
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
419
|
+
if (availableBytes() < bytes + floor) {
|
|
420
|
+
quotaDb.exec("ROLLBACK");
|
|
421
|
+
return "disk";
|
|
422
|
+
}
|
|
423
|
+
quotaDb.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)").run(token, quotaOwner, bytes, expires);
|
|
424
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved + bytes, quotaOwner);
|
|
425
|
+
quotaDb.exec("COMMIT");
|
|
426
|
+
return "ok";
|
|
427
|
+
} catch (error) {
|
|
428
|
+
quotaDb.exec("ROLLBACK");
|
|
429
|
+
throw error;
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const settleR2 = (token: string): void => {
|
|
434
|
+
if (!quotaDb || !quotaOwner) return;
|
|
435
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
436
|
+
try {
|
|
437
|
+
const reservation = quotaDb.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?").get(token);
|
|
438
|
+
if (!reservation) {
|
|
439
|
+
quotaDb.exec("COMMIT");
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
443
|
+
const used = committedR2Bytes();
|
|
444
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
445
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
446
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
447
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
448
|
+
quotaDb.exec("COMMIT");
|
|
449
|
+
} catch (error) {
|
|
450
|
+
quotaDb.exec("ROLLBACK");
|
|
451
|
+
throw error;
|
|
452
|
+
}
|
|
453
|
+
};
|
|
454
|
+
|
|
455
|
+
const recordR2TransferMetric = (field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size"): void => {
|
|
456
|
+
if (!quotaDb || !quotaOwner) return;
|
|
457
|
+
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
458
|
+
};
|
|
459
|
+
|
|
293
460
|
/**
|
|
294
461
|
* Where a KV / R2 / queue binding stores: its resource's own file keyed by
|
|
295
462
|
* `id` when the config bound it to one, else the per-broker `db` partitioned
|
|
@@ -428,39 +595,482 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
428
595
|
* `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
|
|
429
596
|
*/
|
|
430
597
|
const r2MemBlobs = new Map<string, Uint8Array>();
|
|
431
|
-
const
|
|
432
|
-
|
|
598
|
+
const r2MemParts = new Map<string, Uint8Array>();
|
|
599
|
+
const r2BlobId = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
600
|
+
createHash("sha256")
|
|
601
|
+
.update(generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`)
|
|
602
|
+
.digest("hex");
|
|
433
603
|
const r2BlobDir = (store: Database): string | null => {
|
|
434
604
|
const file = store.filename;
|
|
435
605
|
return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
|
|
436
606
|
};
|
|
437
|
-
const r2BlobPath = (store: Database, bucketPart: string, key: string): string | null => {
|
|
607
|
+
const r2BlobPath = (store: Database, bucketPart: string, key: string, generation?: string | null): string | null => {
|
|
438
608
|
const dir = r2BlobDir(store);
|
|
439
|
-
return dir ? join(dir, `${r2BlobId(bucketPart, key)}.blob`) : null;
|
|
609
|
+
return dir ? join(dir, `${r2BlobId(bucketPart, key, generation)}.blob`) : null;
|
|
610
|
+
};
|
|
611
|
+
const r2BlobMemKey = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
612
|
+
generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`;
|
|
613
|
+
const r2WriteBlob = (
|
|
614
|
+
store: Database,
|
|
615
|
+
bucketPart: string,
|
|
616
|
+
key: string,
|
|
617
|
+
bytes: Uint8Array,
|
|
618
|
+
generation: string,
|
|
619
|
+
): void => {
|
|
620
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
621
|
+
if (!path) {
|
|
622
|
+
r2MemBlobs.set(r2BlobMemKey(bucketPart, key, generation), bytes);
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
626
|
+
const temporary = `${path}.tmp`;
|
|
627
|
+
writeFileSync(temporary, bytes);
|
|
628
|
+
renameSync(temporary, path);
|
|
629
|
+
};
|
|
630
|
+
const r2ReadBlob = (
|
|
631
|
+
store: Database,
|
|
632
|
+
bucketPart: string,
|
|
633
|
+
key: string,
|
|
634
|
+
generation?: string | null,
|
|
635
|
+
): Uint8Array | null => {
|
|
636
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
637
|
+
if (!path) return r2MemBlobs.get(r2BlobMemKey(bucketPart, key, generation)) ?? null;
|
|
638
|
+
return existsSync(path) ? readFileSync(path) : null;
|
|
639
|
+
};
|
|
640
|
+
const r2DeleteBlob = (store: Database, bucketPart: string, key: string, generation?: string | null): void => {
|
|
641
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
642
|
+
if (!path) {
|
|
643
|
+
r2MemBlobs.delete(r2BlobMemKey(bucketPart, key, generation));
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
if (existsSync(path)) rmSync(path);
|
|
647
|
+
};
|
|
648
|
+
const r2PartId = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
649
|
+
createHash("sha256").update(`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`).digest("hex");
|
|
650
|
+
const r2PartPath = (
|
|
651
|
+
store: Database,
|
|
652
|
+
bucketPart: string,
|
|
653
|
+
key: string,
|
|
654
|
+
uploadId: string,
|
|
655
|
+
partNumber: number,
|
|
656
|
+
): string | null => {
|
|
657
|
+
const file = store.filename;
|
|
658
|
+
if (!file || file === ":memory:") return null;
|
|
659
|
+
return join(dirname(resolve(file)), "r2-parts", `${r2PartId(bucketPart, key, uploadId, partNumber)}.part`);
|
|
440
660
|
};
|
|
441
|
-
const
|
|
442
|
-
|
|
661
|
+
const r2PartMemKey = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
662
|
+
`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`;
|
|
663
|
+
const r2WritePart = (
|
|
664
|
+
store: Database,
|
|
665
|
+
bucketPart: string,
|
|
666
|
+
key: string,
|
|
667
|
+
uploadId: string,
|
|
668
|
+
partNumber: number,
|
|
669
|
+
bytes: Uint8Array,
|
|
670
|
+
): void => {
|
|
671
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
443
672
|
if (!path) {
|
|
444
|
-
|
|
673
|
+
r2MemParts.set(r2PartMemKey(bucketPart, key, uploadId, partNumber), bytes);
|
|
445
674
|
return;
|
|
446
675
|
}
|
|
447
676
|
mkdirSync(dirname(path), { recursive: true });
|
|
448
|
-
|
|
677
|
+
const temporary = `${path}.tmp`;
|
|
678
|
+
writeFileSync(temporary, bytes);
|
|
679
|
+
renameSync(temporary, path);
|
|
449
680
|
};
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
681
|
+
const r2ReadPart = (
|
|
682
|
+
store: Database,
|
|
683
|
+
bucketPart: string,
|
|
684
|
+
key: string,
|
|
685
|
+
uploadId: string,
|
|
686
|
+
partNumber: number,
|
|
687
|
+
): Uint8Array | null => {
|
|
688
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
689
|
+
if (!path) return r2MemParts.get(r2PartMemKey(bucketPart, key, uploadId, partNumber)) ?? null;
|
|
453
690
|
return existsSync(path) ? readFileSync(path) : null;
|
|
454
691
|
};
|
|
455
|
-
const
|
|
456
|
-
|
|
692
|
+
const r2DeletePart = (
|
|
693
|
+
store: Database,
|
|
694
|
+
bucketPart: string,
|
|
695
|
+
key: string,
|
|
696
|
+
uploadId: string,
|
|
697
|
+
partNumber: number,
|
|
698
|
+
): void => {
|
|
699
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
457
700
|
if (!path) {
|
|
458
|
-
|
|
701
|
+
r2MemParts.delete(r2PartMemKey(bucketPart, key, uploadId, partNumber));
|
|
459
702
|
return;
|
|
460
703
|
}
|
|
461
704
|
if (existsSync(path)) rmSync(path);
|
|
462
705
|
};
|
|
463
706
|
|
|
707
|
+
const requireMultipart = (store: Database, bucket: string, key: string, uploadId: string): R2UploadRow => {
|
|
708
|
+
const row = store
|
|
709
|
+
.query<R2UploadRow, [string, string, string]>(
|
|
710
|
+
"SELECT key, uploaded, http_json, custom_json FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?",
|
|
711
|
+
)
|
|
712
|
+
.get(bucket, key, uploadId);
|
|
713
|
+
if (!row) throw new Error("multipart upload not found");
|
|
714
|
+
return row;
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
const multipartParts = (store: Database, bucket: string, key: string, uploadId: string): R2PartRow[] =>
|
|
718
|
+
store
|
|
719
|
+
.query<R2PartRow, [string, string, string]>(
|
|
720
|
+
"SELECT part_number, size, etag FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ? ORDER BY part_number",
|
|
721
|
+
)
|
|
722
|
+
.all(bucket, key, uploadId);
|
|
723
|
+
|
|
724
|
+
const requestedParts = (value: JsonValue | undefined): Array<{ partNumber: number; etag: string }> => {
|
|
725
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error("complete requires at least one uploaded part");
|
|
726
|
+
return value.map((part) => {
|
|
727
|
+
const parsed = jsonObject(part);
|
|
728
|
+
const partNumber = Number(parsed?.partNumber);
|
|
729
|
+
const etag = isString(parsed?.etag) ? parsed.etag : "";
|
|
730
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000 || !etag) {
|
|
731
|
+
throw new Error("invalid uploaded part");
|
|
732
|
+
}
|
|
733
|
+
return { partNumber, etag };
|
|
734
|
+
});
|
|
735
|
+
};
|
|
736
|
+
|
|
737
|
+
const validateCompletedParts = (stored: R2PartRow[], requested: Array<{ partNumber: number; etag: string }>): void => {
|
|
738
|
+
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
739
|
+
for (let index = 0; index < stored.length; index++) {
|
|
740
|
+
const actual = stored[index];
|
|
741
|
+
const expected = requested[index];
|
|
742
|
+
if (actual.part_number !== expected.partNumber || actual.etag !== expected.etag) {
|
|
743
|
+
throw new Error("uploaded parts do not match");
|
|
744
|
+
}
|
|
745
|
+
if (index > 0 && expected.partNumber <= requested[index - 1].partNumber) {
|
|
746
|
+
throw new Error("uploaded parts must be ordered by partNumber");
|
|
747
|
+
}
|
|
748
|
+
if (index < stored.length - 1 && actual.size !== stored[0].size) {
|
|
749
|
+
throw new Error("all multipart parts except the last must have the same size");
|
|
750
|
+
}
|
|
751
|
+
if (index === stored.length - 1 && stored.length > 1 && actual.size > stored[0].size) {
|
|
752
|
+
throw new Error("the last multipart part cannot be larger than the others");
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
const deleteMultipart = (store: Database, bucket: string, key: string, uploadId: string): void => {
|
|
758
|
+
for (const part of multipartParts(store, bucket, key, uploadId)) {
|
|
759
|
+
r2DeletePart(store, bucket, key, uploadId, part.part_number);
|
|
760
|
+
}
|
|
761
|
+
store.query("DELETE FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
762
|
+
store.query("DELETE FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
const cleanupExpiredMultipart = (store: Database): void => {
|
|
766
|
+
const cutoff = new Date(Date.now() - MULTIPART_EXPIRY_MS).toISOString();
|
|
767
|
+
const expired = store
|
|
768
|
+
.query<R2UploadIdentity, [string]>("SELECT bucket, key, upload_id FROM r2_upload WHERE uploaded < ?")
|
|
769
|
+
.all(cutoff);
|
|
770
|
+
for (const upload of expired) deleteMultipart(store, upload.bucket, upload.key, upload.upload_id);
|
|
771
|
+
};
|
|
772
|
+
|
|
773
|
+
const commitR2Metadata = (
|
|
774
|
+
store: Database,
|
|
775
|
+
bucket: string,
|
|
776
|
+
row: R2Row,
|
|
777
|
+
previous: Pick<R2Row, "blob_id"> | null,
|
|
778
|
+
): void => {
|
|
779
|
+
try {
|
|
780
|
+
store
|
|
781
|
+
.query(
|
|
782
|
+
"INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json, blob_id) " +
|
|
783
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
|
|
784
|
+
"ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, " +
|
|
785
|
+
"custom_json=?7, blob_id=?8",
|
|
786
|
+
)
|
|
787
|
+
.run(bucket, row.key, row.size, row.etag, row.uploaded, row.http_json, row.custom_json, row.blob_id);
|
|
788
|
+
} catch (error) {
|
|
789
|
+
if (previous?.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, row.blob_id);
|
|
790
|
+
throw error;
|
|
791
|
+
}
|
|
792
|
+
if (previous && previous.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, previous.blob_id);
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
const putR2 = (
|
|
796
|
+
store: Database,
|
|
797
|
+
bucket: string,
|
|
798
|
+
key: string,
|
|
799
|
+
bytes: Uint8Array,
|
|
800
|
+
httpJson: string,
|
|
801
|
+
customJson: string,
|
|
802
|
+
): R2Row => {
|
|
803
|
+
const previous = store
|
|
804
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
805
|
+
.get(bucket, key);
|
|
806
|
+
const etag = createHash("sha256").update(bytes).digest("hex");
|
|
807
|
+
const row: R2Row = {
|
|
808
|
+
key,
|
|
809
|
+
size: bytes.byteLength,
|
|
810
|
+
etag,
|
|
811
|
+
uploaded: new Date().toISOString(),
|
|
812
|
+
http_json: httpJson,
|
|
813
|
+
custom_json: customJson,
|
|
814
|
+
blob_id: etag,
|
|
815
|
+
};
|
|
816
|
+
r2WriteBlob(store, bucket, key, bytes, etag);
|
|
817
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
818
|
+
return row;
|
|
819
|
+
};
|
|
820
|
+
|
|
821
|
+
const transferLimit = (): number => {
|
|
822
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
823
|
+
return Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
824
|
+
};
|
|
825
|
+
let activeUploads = 0;
|
|
826
|
+
const maxActiveUploads = 4;
|
|
827
|
+
|
|
828
|
+
const cleanupTransferTemps = (store: Database): void => {
|
|
829
|
+
const directory = r2BlobDir(store);
|
|
830
|
+
if (!directory || !existsSync(directory)) return;
|
|
831
|
+
const now = Date.now();
|
|
832
|
+
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
|
833
|
+
for (const name of readdirSync(directory)) {
|
|
834
|
+
if (!/^[0-9a-f]{24}\.upload\.tmp$/.test(name)) continue;
|
|
835
|
+
const path = join(directory, name);
|
|
836
|
+
try {
|
|
837
|
+
const tokenValue = name.slice(0, 24);
|
|
838
|
+
const ticket = store.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?").get(tokenValue);
|
|
839
|
+
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
840
|
+
} catch (error) {
|
|
841
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
|
|
846
|
+
const transferPath = (binding: string, tokenValue: string): string =>
|
|
847
|
+
`/__sb/r2/transfer/${encodeURIComponent(binding)}/${tokenValue}`;
|
|
848
|
+
|
|
849
|
+
const parseRange = (header: string | null, size: number): { start: number; end: number } | null => {
|
|
850
|
+
if (!header) return { start: 0, end: Math.max(0, size - 1) };
|
|
851
|
+
const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
852
|
+
if (!match || size === 0) return null;
|
|
853
|
+
if (!match[1]) {
|
|
854
|
+
const suffix = Number(match[2]);
|
|
855
|
+
if (!Number.isSafeInteger(suffix) || suffix <= 0) return null;
|
|
856
|
+
return { start: Math.max(0, size - suffix), end: size - 1 };
|
|
857
|
+
}
|
|
858
|
+
const start = Number(match[1]);
|
|
859
|
+
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
860
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return null;
|
|
861
|
+
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
862
|
+
};
|
|
863
|
+
|
|
864
|
+
const transfer = async (request: Request): Promise<Response> => {
|
|
865
|
+
const url = new URL(request.url);
|
|
866
|
+
const match = /^\/__sb\/r2\/transfer\/([A-Z][A-Z0-9_]*)\/([0-9a-f]{24})$/.exec(url.pathname);
|
|
867
|
+
if (!match) return new Response("not found", { status: 404 });
|
|
868
|
+
let store: Database;
|
|
869
|
+
let bucket: string;
|
|
870
|
+
try {
|
|
871
|
+
({ store, part: bucket } = storeFor("r2", requireR2(match[1])));
|
|
872
|
+
} catch {
|
|
873
|
+
return new Response("not found", { status: 404 });
|
|
874
|
+
}
|
|
875
|
+
const tokenValue = match[2];
|
|
876
|
+
const ticket = store
|
|
877
|
+
.query<R2TransferRow, [string]>(
|
|
878
|
+
"SELECT bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json, used " +
|
|
879
|
+
"FROM r2_transfer WHERE token = ?",
|
|
880
|
+
)
|
|
881
|
+
.get(tokenValue);
|
|
882
|
+
const method = request.method.toUpperCase();
|
|
883
|
+
const allowedMethod = ticket?.method === "upload" ? "PUT" : ticket?.method === "download" ? "GET, HEAD" : "";
|
|
884
|
+
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
885
|
+
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
886
|
+
}
|
|
887
|
+
if ((ticket.method === "upload" && method !== "PUT") || (ticket.method === "download" && method !== "GET" && method !== "HEAD")) {
|
|
888
|
+
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
889
|
+
}
|
|
890
|
+
if (ticket.method === "download") {
|
|
891
|
+
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, ticket.key);
|
|
892
|
+
if (!row) return new Response("not found", { status: 404 });
|
|
893
|
+
const etag = `"${row.etag}"`;
|
|
894
|
+
if (request.headers.get("if-none-match") === etag) return new Response(null, { status: 304, headers: { etag } });
|
|
895
|
+
const rangeHeader = request.headers.get("range");
|
|
896
|
+
const range = parseRange(rangeHeader, row.size);
|
|
897
|
+
if (!range) {
|
|
898
|
+
return new Response(null, { status: 416, headers: { "content-range": `bytes */${row.size}`, etag } });
|
|
899
|
+
}
|
|
900
|
+
const path = r2BlobPath(store, bucket, ticket.key, row.blob_id);
|
|
901
|
+
if (!path || !existsSync(path)) return new Response("not found", { status: 404 });
|
|
902
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
903
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
904
|
+
const length = row.size === 0 ? 0 : range.end - range.start + 1;
|
|
905
|
+
const metadata = jsonObject(parseJsonValue(row.http_json));
|
|
906
|
+
const headers = new Headers({
|
|
907
|
+
"accept-ranges": "bytes",
|
|
908
|
+
"content-length": String(length),
|
|
909
|
+
etag,
|
|
910
|
+
});
|
|
911
|
+
if (isString(metadata?.contentType)) headers.set("content-type", metadata.contentType);
|
|
912
|
+
if (rangeHeader) headers.set("content-range", `bytes ${range.start}-${range.end}/${row.size}`);
|
|
913
|
+
const body = method === "HEAD" ? null : Bun.file(path).slice(range.start, range.start + length);
|
|
914
|
+
return new Response(body, { status: rangeHeader ? 206 : 200, headers });
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
const declared = request.headers.has("content-length") ? Number(request.headers.get("content-length")) : NaN;
|
|
918
|
+
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
919
|
+
return new Response("invalid content length", { status: 400 });
|
|
920
|
+
}
|
|
921
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes) return new Response("upload too large", { status: 413 });
|
|
922
|
+
const directory = r2BlobDir(store);
|
|
923
|
+
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
924
|
+
if (activeUploads >= maxActiveUploads) {
|
|
925
|
+
return new Response("too many active uploads", { status: 503, headers: { "retry-after": "5" } });
|
|
926
|
+
}
|
|
927
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
928
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
929
|
+
activeUploads++;
|
|
930
|
+
const temporary = join(directory, `${tokenValue}.upload.tmp`);
|
|
931
|
+
await (async () => {
|
|
932
|
+
mkdirSync(directory, { recursive: true });
|
|
933
|
+
const reservation = await open(temporary, "wx");
|
|
934
|
+
await reservation.close();
|
|
935
|
+
})().catch((error) => {
|
|
936
|
+
activeUploads--;
|
|
937
|
+
throw error;
|
|
938
|
+
});
|
|
939
|
+
const writer = Bun.file(temporary).writer({ highWaterMark: 64 * 1024 });
|
|
940
|
+
const hash = createHash("sha256");
|
|
941
|
+
let size = 0;
|
|
942
|
+
try {
|
|
943
|
+
if (request.body) {
|
|
944
|
+
for await (const chunk of request.body) {
|
|
945
|
+
size += chunk.byteLength;
|
|
946
|
+
if (size > ticket.max_bytes) throw new RangeError("upload too large");
|
|
947
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
948
|
+
if (floor > 0 && availableBytes() < chunk.byteLength + floor) {
|
|
949
|
+
const error = new Error("R2 storage is temporarily unavailable");
|
|
950
|
+
Object.assign(error, { code: "SB_R2_DISK_FLOOR" });
|
|
951
|
+
throw error;
|
|
952
|
+
}
|
|
953
|
+
hash.update(chunk);
|
|
954
|
+
const written = await writer.write(chunk);
|
|
955
|
+
if (written !== chunk.byteLength) throw new Error("upload write was incomplete");
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
await writer.end();
|
|
959
|
+
const persisted = await open(temporary, "r");
|
|
960
|
+
try {
|
|
961
|
+
await persisted.sync();
|
|
962
|
+
} finally {
|
|
963
|
+
await persisted.close();
|
|
964
|
+
}
|
|
965
|
+
const etag = hash.digest("hex");
|
|
966
|
+
if (ticket.expected_sha256 && ticket.expected_sha256 !== etag) {
|
|
967
|
+
rmSync(temporary);
|
|
968
|
+
settleR2(tokenValue);
|
|
969
|
+
return Response.json({ error: "sha256 mismatch", actual: etag }, { status: 422 });
|
|
970
|
+
}
|
|
971
|
+
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
972
|
+
renameSync(temporary, destination);
|
|
973
|
+
const previous = store
|
|
974
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>("SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?")
|
|
975
|
+
.get(bucket, ticket.key);
|
|
976
|
+
const row: R2Row = {
|
|
977
|
+
key: ticket.key,
|
|
978
|
+
size,
|
|
979
|
+
etag,
|
|
980
|
+
uploaded: new Date().toISOString(),
|
|
981
|
+
http_json: ticket.http_json,
|
|
982
|
+
custom_json: ticket.custom_json,
|
|
983
|
+
blob_id: etag,
|
|
984
|
+
};
|
|
985
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
986
|
+
settleR2(tokenValue);
|
|
987
|
+
return Response.json({ object: r2Row(row) }, { status: 201, headers: { etag: `"${etag}"` } });
|
|
988
|
+
} catch (error) {
|
|
989
|
+
try {
|
|
990
|
+
await writer.end();
|
|
991
|
+
} catch {
|
|
992
|
+
// Preserve the original upload failure while releasing the file sink.
|
|
993
|
+
}
|
|
994
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
995
|
+
settleR2(tokenValue);
|
|
996
|
+
if (error instanceof RangeError) {
|
|
997
|
+
recordR2TransferMetric("rejected_size");
|
|
998
|
+
return new Response(error.message, { status: 413 });
|
|
999
|
+
}
|
|
1000
|
+
if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "SB_R2_DISK_FLOOR")) {
|
|
1001
|
+
recordR2TransferMetric("rejected_disk");
|
|
1002
|
+
return new Response("R2 storage is temporarily unavailable", { status: 507 });
|
|
1003
|
+
}
|
|
1004
|
+
throw error;
|
|
1005
|
+
} finally {
|
|
1006
|
+
activeUploads--;
|
|
1007
|
+
}
|
|
1008
|
+
};
|
|
1009
|
+
|
|
1010
|
+
const completeMultipart = (
|
|
1011
|
+
store: Database,
|
|
1012
|
+
bucket: string,
|
|
1013
|
+
key: string,
|
|
1014
|
+
uploadId: string,
|
|
1015
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
1016
|
+
): ReturnType<typeof r2Row> => {
|
|
1017
|
+
const upload = requireMultipart(store, bucket, key, uploadId);
|
|
1018
|
+
const parts = multipartParts(store, bucket, key, uploadId);
|
|
1019
|
+
validateCompletedParts(parts, requested);
|
|
1020
|
+
|
|
1021
|
+
const hash = createHash("sha256");
|
|
1022
|
+
let size = 0;
|
|
1023
|
+
const blobDirectory = r2BlobDir(store);
|
|
1024
|
+
let inMemoryBody: Uint8Array | null = null;
|
|
1025
|
+
let temporary: string | null = null;
|
|
1026
|
+
if (blobDirectory) {
|
|
1027
|
+
mkdirSync(blobDirectory, { recursive: true });
|
|
1028
|
+
temporary = join(blobDirectory, `${uploadId}.tmp`);
|
|
1029
|
+
writeFileSync(temporary, new Uint8Array(0));
|
|
1030
|
+
try {
|
|
1031
|
+
for (const part of parts) {
|
|
1032
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1033
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1034
|
+
appendFileSync(temporary, bytes);
|
|
1035
|
+
hash.update(bytes);
|
|
1036
|
+
size += bytes.byteLength;
|
|
1037
|
+
}
|
|
1038
|
+
} catch (error) {
|
|
1039
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
1040
|
+
throw error;
|
|
1041
|
+
}
|
|
1042
|
+
} else {
|
|
1043
|
+
const chunks: Uint8Array[] = [];
|
|
1044
|
+
for (const part of parts) {
|
|
1045
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1046
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1047
|
+
chunks.push(bytes);
|
|
1048
|
+
hash.update(bytes);
|
|
1049
|
+
size += bytes.byteLength;
|
|
1050
|
+
}
|
|
1051
|
+
inMemoryBody = Buffer.concat(chunks);
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
const etag = hash.digest("hex");
|
|
1055
|
+
const previous = store
|
|
1056
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1057
|
+
.get(bucket, key);
|
|
1058
|
+
if (temporary) renameSync(temporary, r2BlobPath(store, bucket, key, etag)!);
|
|
1059
|
+
else r2WriteBlob(store, bucket, key, inMemoryBody ?? new Uint8Array(0), etag);
|
|
1060
|
+
const row: R2Row = {
|
|
1061
|
+
key,
|
|
1062
|
+
size,
|
|
1063
|
+
etag,
|
|
1064
|
+
uploaded: new Date().toISOString(),
|
|
1065
|
+
http_json: upload.http_json,
|
|
1066
|
+
custom_json: upload.custom_json,
|
|
1067
|
+
blob_id: etag,
|
|
1068
|
+
};
|
|
1069
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
1070
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1071
|
+
return r2Row(row);
|
|
1072
|
+
};
|
|
1073
|
+
|
|
464
1074
|
const r2Row = (r: R2Row) => ({
|
|
465
1075
|
key: r.key,
|
|
466
1076
|
size: r.size,
|
|
@@ -656,24 +1266,107 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
656
1266
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
657
1267
|
const key = str(msg.key);
|
|
658
1268
|
const bytes = new TextEncoder().encode(str(msg.body));
|
|
659
|
-
|
|
1269
|
+
return {
|
|
1270
|
+
ok: true,
|
|
1271
|
+
object: r2Row(
|
|
1272
|
+
putR2(
|
|
1273
|
+
store,
|
|
1274
|
+
bucket,
|
|
1275
|
+
key,
|
|
1276
|
+
bytes,
|
|
1277
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1278
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1279
|
+
),
|
|
1280
|
+
),
|
|
1281
|
+
};
|
|
1282
|
+
}
|
|
1283
|
+
case "r2.multipart.create": {
|
|
1284
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1285
|
+
cleanupExpiredMultipart(store);
|
|
1286
|
+
const key = str(msg.key);
|
|
1287
|
+
const uploadId = str(msg.uploadId) || newId();
|
|
660
1288
|
const uploaded = new Date().toISOString();
|
|
661
|
-
r2WriteBlob(store, bucket, key, bytes);
|
|
662
1289
|
store
|
|
663
1290
|
.query(
|
|
664
|
-
"INSERT INTO
|
|
665
|
-
"ON CONFLICT (bucket, key) DO
|
|
1291
|
+
"INSERT INTO r2_upload (bucket, key, upload_id, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1292
|
+
"ON CONFLICT (bucket, key, upload_id) DO NOTHING",
|
|
666
1293
|
)
|
|
667
1294
|
.run(
|
|
668
1295
|
bucket,
|
|
669
1296
|
key,
|
|
670
|
-
|
|
671
|
-
etag,
|
|
1297
|
+
uploadId,
|
|
672
1298
|
uploaded,
|
|
673
1299
|
JSON.stringify(msg.httpMetadata ?? {}),
|
|
674
1300
|
JSON.stringify(msg.customMetadata ?? {}),
|
|
675
1301
|
);
|
|
676
|
-
return { ok: true,
|
|
1302
|
+
return { ok: true, key, uploadId };
|
|
1303
|
+
}
|
|
1304
|
+
case "r2.transfer.create": {
|
|
1305
|
+
const binding = requireR2(msg.bucket);
|
|
1306
|
+
const { store, part: bucket } = storeFor("r2", binding);
|
|
1307
|
+
const key = str(msg.key);
|
|
1308
|
+
const method = msg.method === "download" ? "download" : msg.method === "upload" ? "upload" : "";
|
|
1309
|
+
if (!method) throw new Error("transfer method must be upload or download");
|
|
1310
|
+
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1311
|
+
const expires = Date.now() + expiresIn * 1000;
|
|
1312
|
+
const maxBytes =
|
|
1313
|
+
method === "upload"
|
|
1314
|
+
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1315
|
+
: 0;
|
|
1316
|
+
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1317
|
+
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1318
|
+
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
1319
|
+
}
|
|
1320
|
+
store.query("DELETE FROM r2_transfer WHERE expires < ? OR used = 1").run(Date.now());
|
|
1321
|
+
cleanupTransferTemps(store);
|
|
1322
|
+
const tokenValue = randomBytes(12).toString("hex");
|
|
1323
|
+
const reservation = method === "upload" ? reserveR2(tokenValue, maxBytes, expires) : "ok";
|
|
1324
|
+
if (reservation === "quota") {
|
|
1325
|
+
recordR2TransferMetric("rejected_quota");
|
|
1326
|
+
throw new Error("R2 account quota would be exceeded");
|
|
1327
|
+
}
|
|
1328
|
+
if (reservation === "disk") {
|
|
1329
|
+
recordR2TransferMetric("rejected_disk");
|
|
1330
|
+
throw new Error("R2 storage is temporarily unavailable");
|
|
1331
|
+
}
|
|
1332
|
+
try {
|
|
1333
|
+
store
|
|
1334
|
+
.query(
|
|
1335
|
+
"INSERT INTO r2_transfer (token, bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json) " +
|
|
1336
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
|
|
1337
|
+
)
|
|
1338
|
+
.run(
|
|
1339
|
+
tokenValue,
|
|
1340
|
+
bucket,
|
|
1341
|
+
key,
|
|
1342
|
+
method,
|
|
1343
|
+
expires,
|
|
1344
|
+
maxBytes,
|
|
1345
|
+
expectedSha256,
|
|
1346
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1347
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1348
|
+
);
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
if (reservation === "ok" && method === "upload") settleR2(tokenValue);
|
|
1351
|
+
throw error;
|
|
1352
|
+
}
|
|
1353
|
+
recordR2TransferMetric("tickets_issued");
|
|
1354
|
+
return { ok: true, url: transferPath(binding, tokenValue), expiresAt: new Date(expires).toISOString() };
|
|
1355
|
+
}
|
|
1356
|
+
case "r2.multipart.complete": {
|
|
1357
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1358
|
+
const key = str(msg.key);
|
|
1359
|
+
const uploadId = str(msg.uploadId);
|
|
1360
|
+
const object = completeMultipart(store, bucket, key, uploadId, requestedParts(msg.parts));
|
|
1361
|
+
return { ok: true, object };
|
|
1362
|
+
}
|
|
1363
|
+
case "r2.multipart.abort": {
|
|
1364
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1365
|
+
const key = str(msg.key);
|
|
1366
|
+
const uploadId = str(msg.uploadId);
|
|
1367
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
1368
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1369
|
+
return { ok: true };
|
|
677
1370
|
}
|
|
678
1371
|
case "r2.get":
|
|
679
1372
|
case "r2.head": {
|
|
@@ -681,15 +1374,19 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
681
1374
|
const key = str(msg.key);
|
|
682
1375
|
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
683
1376
|
if (!row) return { ok: true, found: false };
|
|
684
|
-
const body =
|
|
685
|
-
|
|
1377
|
+
const body = msg.op === "r2.get"
|
|
1378
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1379
|
+
: undefined;
|
|
686
1380
|
return { ok: true, found: true, object: r2Row(row), body };
|
|
687
1381
|
}
|
|
688
1382
|
case "r2.delete": {
|
|
689
1383
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
690
1384
|
const key = str(msg.key);
|
|
1385
|
+
const row = store
|
|
1386
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1387
|
+
.get(bucket, key);
|
|
691
1388
|
store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
|
|
692
|
-
r2DeleteBlob(store, bucket, key);
|
|
1389
|
+
if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
|
|
693
1390
|
return { ok: true };
|
|
694
1391
|
}
|
|
695
1392
|
case "r2.list": {
|
|
@@ -876,24 +1573,35 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
876
1573
|
// second full-object copy through the database's own page cache.
|
|
877
1574
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
878
1575
|
const key = str(msg.key);
|
|
1576
|
+
const row = putR2(
|
|
1577
|
+
store,
|
|
1578
|
+
bucket,
|
|
1579
|
+
key,
|
|
1580
|
+
binary,
|
|
1581
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1582
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1583
|
+
);
|
|
1584
|
+
return { reply: { ok: true, object: r2Row(row) } };
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
if (msg.op === "r2.multipart.put") {
|
|
1588
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1589
|
+
const key = str(msg.key);
|
|
1590
|
+
const uploadId = str(msg.uploadId);
|
|
1591
|
+
const partNumber = Number(msg.partNumber);
|
|
1592
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
|
|
1593
|
+
throw new Error("partNumber must be an integer between 1 and 10000");
|
|
1594
|
+
}
|
|
1595
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
879
1596
|
const etag = createHash("sha256").update(binary).digest("hex");
|
|
880
|
-
|
|
881
|
-
r2WriteBlob(store, bucket, key, binary);
|
|
1597
|
+
r2WritePart(store, bucket, key, uploadId, partNumber, binary);
|
|
882
1598
|
store
|
|
883
1599
|
.query(
|
|
884
|
-
"INSERT INTO
|
|
885
|
-
"ON CONFLICT (bucket, key) DO UPDATE SET size=?
|
|
1600
|
+
"INSERT INTO r2_part (bucket, key, upload_id, part_number, size, etag) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1601
|
+
"ON CONFLICT (bucket, key, upload_id, part_number) DO UPDATE SET size=?5, etag=?6",
|
|
886
1602
|
)
|
|
887
|
-
.run(
|
|
888
|
-
|
|
889
|
-
key,
|
|
890
|
-
binary.byteLength,
|
|
891
|
-
etag,
|
|
892
|
-
uploaded,
|
|
893
|
-
JSON.stringify(msg.httpMetadata ?? {}),
|
|
894
|
-
JSON.stringify(msg.customMetadata ?? {}),
|
|
895
|
-
);
|
|
896
|
-
return { reply: { ok: true, object: { key, size: binary.byteLength, etag, uploaded } } };
|
|
1603
|
+
.run(bucket, key, uploadId, partNumber, binary.byteLength, etag);
|
|
1604
|
+
return { reply: { ok: true, part: { partNumber, etag } } };
|
|
897
1605
|
}
|
|
898
1606
|
|
|
899
1607
|
if (msg.op === "r2.get") {
|
|
@@ -901,7 +1609,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
901
1609
|
const key = str(msg.key);
|
|
902
1610
|
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
903
1611
|
if (!row) return { reply: { ok: true, found: false } };
|
|
904
|
-
return {
|
|
1612
|
+
return {
|
|
1613
|
+
reply: { ok: true, found: true, object: r2Row(row) },
|
|
1614
|
+
bytes: r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0),
|
|
1615
|
+
};
|
|
905
1616
|
}
|
|
906
1617
|
|
|
907
1618
|
if (msg.op === "assets.get") {
|
|
@@ -939,7 +1650,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
939
1650
|
if (!msg) throw new Error("request frame was not a JSON object");
|
|
940
1651
|
if (token && str(msg.token) !== token) return frameOf(encodeV1({ ok: false, error: "unauthorized" }));
|
|
941
1652
|
const binary = new Uint8Array(payload.subarray(5 + jsonLen));
|
|
1653
|
+
const id = isSafeInteger(msg.id) ? msg.id : null;
|
|
1654
|
+
const replayKey = id === null ? null : `${str(msg.op)}:${id}`;
|
|
1655
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) {
|
|
1656
|
+
const seen = replayed.get(replayKey);
|
|
1657
|
+
if (seen) return frameOf(encodeV1(seen));
|
|
1658
|
+
}
|
|
942
1659
|
const { reply, bytes } = await dispatchBinary(msg, binary);
|
|
1660
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
|
|
943
1661
|
return frameOf(encodeV1(reply, bytes));
|
|
944
1662
|
} catch (e) {
|
|
945
1663
|
return frameOf(encodeV1({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
@@ -966,6 +1684,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
966
1684
|
"d1.backup",
|
|
967
1685
|
"ratelimit.check",
|
|
968
1686
|
"r2.put",
|
|
1687
|
+
"r2.multipart.create",
|
|
1688
|
+
"r2.multipart.put",
|
|
1689
|
+
"r2.multipart.complete",
|
|
1690
|
+
"r2.multipart.abort",
|
|
1691
|
+
"r2.transfer.create",
|
|
969
1692
|
"r2.delete",
|
|
970
1693
|
"do.storage.put",
|
|
971
1694
|
"do.storage.delete",
|
|
@@ -979,6 +1702,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
979
1702
|
* milliseconds, so this only has to outlive a reconnect, not a session. */
|
|
980
1703
|
const REPLAY_WINDOW = 256;
|
|
981
1704
|
|
|
1705
|
+
const rememberReplay = (key: string, reply: Frame): void => {
|
|
1706
|
+
replayed.set(key, reply);
|
|
1707
|
+
if (replayed.size > REPLAY_WINDOW) {
|
|
1708
|
+
const oldest = replayed.keys().next().value;
|
|
1709
|
+
if (oldest !== undefined) replayed.delete(oldest);
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
|
|
982
1713
|
/**
|
|
983
1714
|
* Run `msg`, or replay what it answered last time. Keyed by the id the sprout
|
|
984
1715
|
* put on the request; a request without one is always run.
|
|
@@ -991,12 +1722,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
991
1722
|
const seen = replayed.get(key);
|
|
992
1723
|
if (seen) return seen;
|
|
993
1724
|
const reply = await dispatch(msg);
|
|
994
|
-
|
|
995
|
-
// Insertion-ordered, so the oldest key is the first one out.
|
|
996
|
-
if (replayed.size > REPLAY_WINDOW) {
|
|
997
|
-
const oldest = replayed.keys().next().value;
|
|
998
|
-
if (oldest !== undefined) replayed.delete(oldest);
|
|
999
|
-
}
|
|
1725
|
+
rememberReplay(key, reply);
|
|
1000
1726
|
return reply;
|
|
1001
1727
|
}
|
|
1002
1728
|
|
|
@@ -1143,6 +1869,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1143
1869
|
dispatch,
|
|
1144
1870
|
handlePayload,
|
|
1145
1871
|
handleFrame,
|
|
1872
|
+
transfer,
|
|
1146
1873
|
close: () => {
|
|
1147
1874
|
for (const t of timers) clearInterval(t);
|
|
1148
1875
|
for (const conn of d1Conns.values()) conn.close();
|
|
@@ -1230,17 +1957,33 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
1230
1957
|
},
|
|
1231
1958
|
},
|
|
1232
1959
|
});
|
|
1233
|
-
return { port: server.port, stop: () => server.stop(true) };
|
|
1960
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
/** Start the HTTP listener that serves short-lived direct R2 transfer tickets. */
|
|
1964
|
+
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
1965
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
1966
|
+
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
1967
|
+
const server = Bun.serve({ hostname, port, maxRequestBodySize, fetch(request, server) {
|
|
1968
|
+
server.timeout(request, 255);
|
|
1969
|
+
return broker.transfer(request);
|
|
1970
|
+
} });
|
|
1971
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1234
1972
|
}
|
|
1235
1973
|
|
|
1236
1974
|
if (import.meta.main) {
|
|
1237
1975
|
const { values } = parseArgs({
|
|
1238
1976
|
options: {
|
|
1239
1977
|
port: { type: "string" },
|
|
1978
|
+
"transfer-port": { type: "string" },
|
|
1240
1979
|
token: { type: "string" },
|
|
1241
1980
|
db: { type: "string" },
|
|
1242
1981
|
"data-dir": { type: "string" },
|
|
1243
1982
|
"resource-dir": { type: "string" },
|
|
1983
|
+
"owner-id": { type: "string" },
|
|
1984
|
+
"r2-quota-bytes": { type: "string" },
|
|
1985
|
+
"r2-resource-ids": { type: "string" },
|
|
1986
|
+
"r2-min-free-bytes": { type: "string" },
|
|
1244
1987
|
bindings: { type: "string" },
|
|
1245
1988
|
secrets: { type: "string" },
|
|
1246
1989
|
"sprout-url": { type: "string" },
|
|
@@ -1289,6 +2032,10 @@ if (import.meta.main) {
|
|
|
1289
2032
|
db: values.db,
|
|
1290
2033
|
dataDir: values["data-dir"],
|
|
1291
2034
|
resourceDir: values["resource-dir"],
|
|
2035
|
+
ownerId: values["owner-id"],
|
|
2036
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2037
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2038
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
1292
2039
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
1293
2040
|
bindings,
|
|
1294
2041
|
secrets,
|
|
@@ -1299,7 +2046,9 @@ if (import.meta.main) {
|
|
|
1299
2046
|
dispatchEnabled: () => dispatchEnabled,
|
|
1300
2047
|
});
|
|
1301
2048
|
const { port } = listen(broker, "127.0.0.1", Number(values.port ?? process.env.SB_BROKER_PORT ?? 0));
|
|
2049
|
+
const transferPort = Number(values["transfer-port"] ?? process.env.SB_R2_TRANSFER_PORT ?? 0);
|
|
2050
|
+
const transferServer = transferPort > 0 ? listenTransfers(broker, "127.0.0.1", transferPort) : null;
|
|
1302
2051
|
console.log(
|
|
1303
|
-
`sproutboat broker: 127.0.0.1:${port} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
2052
|
+
`sproutboat broker: 127.0.0.1:${port} transfer=${transferServer?.port ?? "off"} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
1304
2053
|
);
|
|
1305
2054
|
}
|