@sproutboat/wire 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/package.json +6 -1
- package/src/broker.test.ts +383 -1
- package/src/broker.ts +848 -61
package/src/broker.ts
CHANGED
|
@@ -18,8 +18,9 @@
|
|
|
18
18
|
* the only SSRF control today.
|
|
19
19
|
*/
|
|
20
20
|
import { Database, type Statement } from "bun:sqlite";
|
|
21
|
-
import { createHash } from "node:crypto";
|
|
22
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
21
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
22
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statfsSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { open } from "node:fs/promises";
|
|
23
24
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
24
25
|
import { parseArgs } from "node:util";
|
|
25
26
|
import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
|
|
@@ -84,6 +85,16 @@ export type BrokerOptions = {
|
|
|
84
85
|
* data outlives a redeploy.
|
|
85
86
|
*/
|
|
86
87
|
resourceDir?: string;
|
|
88
|
+
/** Account whose R2 resources this broker may mutate. */
|
|
89
|
+
ownerId?: string;
|
|
90
|
+
/** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
|
|
91
|
+
r2QuotaBytes?: number;
|
|
92
|
+
/** Resource ids belonging to ownerId, used to seed the shared R2 usage ledger. */
|
|
93
|
+
r2ResourceIds?: string[];
|
|
94
|
+
/** Free disk capacity retained while direct R2 uploads are accepted. */
|
|
95
|
+
r2MinFreeBytes?: number;
|
|
96
|
+
/** Test hook for checking the filesystem capacity available to temporary uploads. */
|
|
97
|
+
freeBytes?: (path: string) => number;
|
|
87
98
|
token?: string;
|
|
88
99
|
bindings?: Partial<Bindings>;
|
|
89
100
|
secrets?: Record<string, string>;
|
|
@@ -110,14 +121,33 @@ type D1Result = {
|
|
|
110
121
|
};
|
|
111
122
|
type R2Row = {
|
|
112
123
|
key: string;
|
|
113
|
-
/** TEXT from a v0 write, BLOB from a v1 one; sqlite hands each back as-is. */
|
|
114
|
-
body: string | Uint8Array;
|
|
115
124
|
size: number;
|
|
116
125
|
etag: string;
|
|
117
126
|
uploaded: string;
|
|
118
127
|
http_json: string;
|
|
119
128
|
custom_json: string;
|
|
129
|
+
blob_id: string | null;
|
|
120
130
|
};
|
|
131
|
+
type R2UploadRow = {
|
|
132
|
+
key: string;
|
|
133
|
+
uploaded: string;
|
|
134
|
+
http_json: string;
|
|
135
|
+
custom_json: string;
|
|
136
|
+
};
|
|
137
|
+
type R2PartRow = { part_number: number; size: number; etag: string };
|
|
138
|
+
type R2UploadIdentity = { bucket: string; key: string; upload_id: string };
|
|
139
|
+
type R2TransferRow = {
|
|
140
|
+
bucket: string;
|
|
141
|
+
key: string;
|
|
142
|
+
method: string;
|
|
143
|
+
expires: number;
|
|
144
|
+
max_bytes: number;
|
|
145
|
+
expected_sha256: string;
|
|
146
|
+
http_json: string;
|
|
147
|
+
custom_json: string;
|
|
148
|
+
used: number;
|
|
149
|
+
};
|
|
150
|
+
const MULTIPART_EXPIRY_MS = 7 * 24 * 60 * 60 * 1000;
|
|
121
151
|
const isNumber = (v: JsonValue | undefined): v is number => Number.isFinite(v);
|
|
122
152
|
const sqlParams = (v: JsonValue | undefined): SqlParam[] => {
|
|
123
153
|
if (!Array.isArray(v)) return [];
|
|
@@ -146,6 +176,8 @@ export type Broker = {
|
|
|
146
176
|
* negotiation.
|
|
147
177
|
*/
|
|
148
178
|
handleFrame(payload: Buffer): Promise<Buffer>;
|
|
179
|
+
/** Serve one reserved direct R2 upload or download request. */
|
|
180
|
+
transfer(request: Request): Promise<Response>;
|
|
149
181
|
close(): void;
|
|
150
182
|
};
|
|
151
183
|
|
|
@@ -234,11 +266,35 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
234
266
|
conn.exec(
|
|
235
267
|
"CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (ns, key))",
|
|
236
268
|
);
|
|
269
|
+
// #56 — no `body` column: an object's bytes live in their own file under
|
|
270
|
+
// `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
|
|
271
|
+
// Doubling every object through SQLite's own page cache on top of the
|
|
272
|
+
// buffer it already arrived in was the real cost driving #56's "large
|
|
273
|
+
// uploads make memory skyrocket" complaint, not just the JSON-escaping
|
|
274
|
+
// #63 already fixed.
|
|
237
275
|
conn.exec(
|
|
238
|
-
"CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL,
|
|
276
|
+
"CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
|
|
239
277
|
"etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
278
|
+
"blob_id TEXT, " +
|
|
240
279
|
"PRIMARY KEY (bucket, key))",
|
|
241
280
|
);
|
|
281
|
+
const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
|
|
282
|
+
if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
|
|
283
|
+
conn.exec(
|
|
284
|
+
"CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
285
|
+
"uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
286
|
+
"PRIMARY KEY (bucket, key, upload_id))",
|
|
287
|
+
);
|
|
288
|
+
conn.exec(
|
|
289
|
+
"CREATE TABLE IF NOT EXISTS r2_part (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
290
|
+
"part_number INTEGER NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, " +
|
|
291
|
+
"PRIMARY KEY (bucket, key, upload_id, part_number))",
|
|
292
|
+
);
|
|
293
|
+
conn.exec(
|
|
294
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer (token TEXT PRIMARY KEY, bucket TEXT NOT NULL, key TEXT NOT NULL, " +
|
|
295
|
+
"method TEXT NOT NULL, expires INTEGER NOT NULL, max_bytes INTEGER NOT NULL, expected_sha256 TEXT NOT NULL DEFAULT '', " +
|
|
296
|
+
"http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', used INTEGER NOT NULL DEFAULT 0)",
|
|
297
|
+
);
|
|
242
298
|
conn.exec(
|
|
243
299
|
"CREATE TABLE IF NOT EXISTS mq (queue TEXT NOT NULL, id TEXT PRIMARY KEY, body TEXT NOT NULL, " +
|
|
244
300
|
"visible_at INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, dead INTEGER NOT NULL DEFAULT 0)",
|
|
@@ -286,6 +342,121 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
286
342
|
return conn;
|
|
287
343
|
};
|
|
288
344
|
|
|
345
|
+
const quotaOwner = opts.ownerId;
|
|
346
|
+
const quotaBytes = opts.r2QuotaBytes && opts.r2QuotaBytes > 0 ? opts.r2QuotaBytes : null;
|
|
347
|
+
const quotaDirectory = resourceDir;
|
|
348
|
+
if (quotaOwner && quotaDirectory && quotaBytes) mkdirSync(quotaDirectory, { recursive: true });
|
|
349
|
+
// One ledger for every account, shared by the brokers of all its deployments.
|
|
350
|
+
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
351
|
+
// requests cannot each independently decide that the same remaining bytes fit.
|
|
352
|
+
const quotaDb = quotaOwner && quotaDirectory && quotaBytes
|
|
353
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
354
|
+
: null;
|
|
355
|
+
if (quotaDb) {
|
|
356
|
+
const owner = quotaOwner!;
|
|
357
|
+
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
358
|
+
quotaDb.exec(
|
|
359
|
+
"CREATE TABLE IF NOT EXISTS r2_account_quota (owner TEXT PRIMARY KEY, used_bytes INTEGER NOT NULL, reserved_bytes INTEGER NOT NULL)",
|
|
360
|
+
);
|
|
361
|
+
quotaDb.exec(
|
|
362
|
+
"CREATE TABLE IF NOT EXISTS r2_quota_reservation (token TEXT PRIMARY KEY, owner TEXT NOT NULL, bytes INTEGER NOT NULL, expires INTEGER NOT NULL)",
|
|
363
|
+
);
|
|
364
|
+
quotaDb.exec(
|
|
365
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
366
|
+
);
|
|
367
|
+
const existing = quotaDb.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?").get(owner);
|
|
368
|
+
if (!existing) {
|
|
369
|
+
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
370
|
+
}
|
|
371
|
+
quotaDb.query("INSERT OR IGNORE INTO r2_transfer_metric (owner) VALUES (?)").run(owner);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
const committedR2Bytes = (): number => {
|
|
375
|
+
if (!quotaDirectory) return 0;
|
|
376
|
+
let used = 0;
|
|
377
|
+
for (const id of opts.r2ResourceIds ?? []) {
|
|
378
|
+
// Open through resourceDb so older resource files gain the multipart
|
|
379
|
+
// tables before the aggregate is queried.
|
|
380
|
+
const store = resourceDb(id);
|
|
381
|
+
used += store.query<{ bytes: number }, []>(
|
|
382
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
383
|
+
).get()?.bytes ?? 0;
|
|
384
|
+
}
|
|
385
|
+
return used;
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
const availableBytes = (): number => {
|
|
389
|
+
if (!quotaDirectory) return Number.POSITIVE_INFINITY;
|
|
390
|
+
if (opts.freeBytes) return opts.freeBytes(quotaDirectory);
|
|
391
|
+
const stats = statfsSync(quotaDirectory);
|
|
392
|
+
return Number(stats.bavail) * Number(stats.bsize);
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
396
|
+
if (!quotaDb || !quotaOwner) return;
|
|
397
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
398
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
399
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
400
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
401
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
const reserveR2 = (token: string, bytes: number, expires: number): "ok" | "quota" | "disk" => {
|
|
405
|
+
if (!quotaDb || !quotaOwner || !quotaBytes) return "ok";
|
|
406
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
407
|
+
try {
|
|
408
|
+
releaseExpiredR2Reservations();
|
|
409
|
+
const used = committedR2Bytes();
|
|
410
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
411
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
412
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
413
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
414
|
+
if (used + reserved + bytes > quotaBytes) {
|
|
415
|
+
quotaDb.exec("ROLLBACK");
|
|
416
|
+
return "quota";
|
|
417
|
+
}
|
|
418
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
419
|
+
if (availableBytes() < bytes + floor) {
|
|
420
|
+
quotaDb.exec("ROLLBACK");
|
|
421
|
+
return "disk";
|
|
422
|
+
}
|
|
423
|
+
quotaDb.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)").run(token, quotaOwner, bytes, expires);
|
|
424
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved + bytes, quotaOwner);
|
|
425
|
+
quotaDb.exec("COMMIT");
|
|
426
|
+
return "ok";
|
|
427
|
+
} catch (error) {
|
|
428
|
+
quotaDb.exec("ROLLBACK");
|
|
429
|
+
throw error;
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const settleR2 = (token: string): void => {
|
|
434
|
+
if (!quotaDb || !quotaOwner) return;
|
|
435
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
436
|
+
try {
|
|
437
|
+
const reservation = quotaDb.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?").get(token);
|
|
438
|
+
if (!reservation) {
|
|
439
|
+
quotaDb.exec("COMMIT");
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
443
|
+
const used = committedR2Bytes();
|
|
444
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
445
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
446
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
447
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
448
|
+
quotaDb.exec("COMMIT");
|
|
449
|
+
} catch (error) {
|
|
450
|
+
quotaDb.exec("ROLLBACK");
|
|
451
|
+
throw error;
|
|
452
|
+
}
|
|
453
|
+
};
|
|
454
|
+
|
|
455
|
+
const recordR2TransferMetric = (field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size"): void => {
|
|
456
|
+
if (!quotaDb || !quotaOwner) return;
|
|
457
|
+
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
458
|
+
};
|
|
459
|
+
|
|
289
460
|
/**
|
|
290
461
|
* Where a KV / R2 / queue binding stores: its resource's own file keyed by
|
|
291
462
|
* `id` when the config bound it to one, else the per-broker `db` partitioned
|
|
@@ -414,15 +585,491 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
414
585
|
return { path: dest, bytes: statSync(dest).size };
|
|
415
586
|
};
|
|
416
587
|
|
|
417
|
-
/**
|
|
418
|
-
|
|
419
|
-
|
|
588
|
+
/**
|
|
589
|
+
* #56 — an object's bytes never enter SQLite; they live in their own file,
|
|
590
|
+
* named by a hash of bucket+key (so unicode keys and directory traversal
|
|
591
|
+
* are both non-issues), colocated with whichever store file holds that
|
|
592
|
+
* bucket's metadata — the main `db` for a bare-string binding, the
|
|
593
|
+
* resource's own file under `resourceDir` for an account-level one (#74) —
|
|
594
|
+
* so blobs persist across a redeploy exactly when that metadata does.
|
|
595
|
+
* `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
|
|
596
|
+
*/
|
|
597
|
+
const r2MemBlobs = new Map<string, Uint8Array>();
|
|
598
|
+
const r2MemParts = new Map<string, Uint8Array>();
|
|
599
|
+
const r2BlobId = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
600
|
+
createHash("sha256")
|
|
601
|
+
.update(generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`)
|
|
602
|
+
.digest("hex");
|
|
603
|
+
const r2BlobDir = (store: Database): string | null => {
|
|
604
|
+
const file = store.filename;
|
|
605
|
+
return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
|
|
606
|
+
};
|
|
607
|
+
const r2BlobPath = (store: Database, bucketPart: string, key: string, generation?: string | null): string | null => {
|
|
608
|
+
const dir = r2BlobDir(store);
|
|
609
|
+
return dir ? join(dir, `${r2BlobId(bucketPart, key, generation)}.blob`) : null;
|
|
610
|
+
};
|
|
611
|
+
const r2BlobMemKey = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
612
|
+
generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`;
|
|
613
|
+
const r2WriteBlob = (
|
|
614
|
+
store: Database,
|
|
615
|
+
bucketPart: string,
|
|
616
|
+
key: string,
|
|
617
|
+
bytes: Uint8Array,
|
|
618
|
+
generation: string,
|
|
619
|
+
): void => {
|
|
620
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
621
|
+
if (!path) {
|
|
622
|
+
r2MemBlobs.set(r2BlobMemKey(bucketPart, key, generation), bytes);
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
626
|
+
const temporary = `${path}.tmp`;
|
|
627
|
+
writeFileSync(temporary, bytes);
|
|
628
|
+
renameSync(temporary, path);
|
|
629
|
+
};
|
|
630
|
+
const r2ReadBlob = (
|
|
631
|
+
store: Database,
|
|
632
|
+
bucketPart: string,
|
|
633
|
+
key: string,
|
|
634
|
+
generation?: string | null,
|
|
635
|
+
): Uint8Array | null => {
|
|
636
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
637
|
+
if (!path) return r2MemBlobs.get(r2BlobMemKey(bucketPart, key, generation)) ?? null;
|
|
638
|
+
return existsSync(path) ? readFileSync(path) : null;
|
|
639
|
+
};
|
|
640
|
+
const r2DeleteBlob = (store: Database, bucketPart: string, key: string, generation?: string | null): void => {
|
|
641
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
642
|
+
if (!path) {
|
|
643
|
+
r2MemBlobs.delete(r2BlobMemKey(bucketPart, key, generation));
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
if (existsSync(path)) rmSync(path);
|
|
647
|
+
};
|
|
648
|
+
const r2PartId = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
649
|
+
createHash("sha256").update(`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`).digest("hex");
|
|
650
|
+
const r2PartPath = (
|
|
651
|
+
store: Database,
|
|
652
|
+
bucketPart: string,
|
|
653
|
+
key: string,
|
|
654
|
+
uploadId: string,
|
|
655
|
+
partNumber: number,
|
|
656
|
+
): string | null => {
|
|
657
|
+
const file = store.filename;
|
|
658
|
+
if (!file || file === ":memory:") return null;
|
|
659
|
+
return join(dirname(resolve(file)), "r2-parts", `${r2PartId(bucketPart, key, uploadId, partNumber)}.part`);
|
|
660
|
+
};
|
|
661
|
+
const r2PartMemKey = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
662
|
+
`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`;
|
|
663
|
+
const r2WritePart = (
|
|
664
|
+
store: Database,
|
|
665
|
+
bucketPart: string,
|
|
666
|
+
key: string,
|
|
667
|
+
uploadId: string,
|
|
668
|
+
partNumber: number,
|
|
669
|
+
bytes: Uint8Array,
|
|
670
|
+
): void => {
|
|
671
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
672
|
+
if (!path) {
|
|
673
|
+
r2MemParts.set(r2PartMemKey(bucketPart, key, uploadId, partNumber), bytes);
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
677
|
+
const temporary = `${path}.tmp`;
|
|
678
|
+
writeFileSync(temporary, bytes);
|
|
679
|
+
renameSync(temporary, path);
|
|
680
|
+
};
|
|
681
|
+
const r2ReadPart = (
|
|
682
|
+
store: Database,
|
|
683
|
+
bucketPart: string,
|
|
684
|
+
key: string,
|
|
685
|
+
uploadId: string,
|
|
686
|
+
partNumber: number,
|
|
687
|
+
): Uint8Array | null => {
|
|
688
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
689
|
+
if (!path) return r2MemParts.get(r2PartMemKey(bucketPart, key, uploadId, partNumber)) ?? null;
|
|
690
|
+
return existsSync(path) ? readFileSync(path) : null;
|
|
691
|
+
};
|
|
692
|
+
const r2DeletePart = (
|
|
693
|
+
store: Database,
|
|
694
|
+
bucketPart: string,
|
|
695
|
+
key: string,
|
|
696
|
+
uploadId: string,
|
|
697
|
+
partNumber: number,
|
|
698
|
+
): void => {
|
|
699
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
700
|
+
if (!path) {
|
|
701
|
+
r2MemParts.delete(r2PartMemKey(bucketPart, key, uploadId, partNumber));
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
if (existsSync(path)) rmSync(path);
|
|
705
|
+
};
|
|
706
|
+
|
|
707
|
+
const requireMultipart = (store: Database, bucket: string, key: string, uploadId: string): R2UploadRow => {
|
|
708
|
+
const row = store
|
|
709
|
+
.query<R2UploadRow, [string, string, string]>(
|
|
710
|
+
"SELECT key, uploaded, http_json, custom_json FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?",
|
|
711
|
+
)
|
|
712
|
+
.get(bucket, key, uploadId);
|
|
713
|
+
if (!row) throw new Error("multipart upload not found");
|
|
714
|
+
return row;
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
const multipartParts = (store: Database, bucket: string, key: string, uploadId: string): R2PartRow[] =>
|
|
718
|
+
store
|
|
719
|
+
.query<R2PartRow, [string, string, string]>(
|
|
720
|
+
"SELECT part_number, size, etag FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ? ORDER BY part_number",
|
|
721
|
+
)
|
|
722
|
+
.all(bucket, key, uploadId);
|
|
723
|
+
|
|
724
|
+
const requestedParts = (value: JsonValue | undefined): Array<{ partNumber: number; etag: string }> => {
|
|
725
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error("complete requires at least one uploaded part");
|
|
726
|
+
return value.map((part) => {
|
|
727
|
+
const parsed = jsonObject(part);
|
|
728
|
+
const partNumber = Number(parsed?.partNumber);
|
|
729
|
+
const etag = isString(parsed?.etag) ? parsed.etag : "";
|
|
730
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000 || !etag) {
|
|
731
|
+
throw new Error("invalid uploaded part");
|
|
732
|
+
}
|
|
733
|
+
return { partNumber, etag };
|
|
734
|
+
});
|
|
735
|
+
};
|
|
736
|
+
|
|
737
|
+
const validateCompletedParts = (stored: R2PartRow[], requested: Array<{ partNumber: number; etag: string }>): void => {
|
|
738
|
+
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
739
|
+
for (let index = 0; index < stored.length; index++) {
|
|
740
|
+
const actual = stored[index];
|
|
741
|
+
const expected = requested[index];
|
|
742
|
+
if (actual.part_number !== expected.partNumber || actual.etag !== expected.etag) {
|
|
743
|
+
throw new Error("uploaded parts do not match");
|
|
744
|
+
}
|
|
745
|
+
if (index > 0 && expected.partNumber <= requested[index - 1].partNumber) {
|
|
746
|
+
throw new Error("uploaded parts must be ordered by partNumber");
|
|
747
|
+
}
|
|
748
|
+
if (index < stored.length - 1 && actual.size !== stored[0].size) {
|
|
749
|
+
throw new Error("all multipart parts except the last must have the same size");
|
|
750
|
+
}
|
|
751
|
+
if (index === stored.length - 1 && stored.length > 1 && actual.size > stored[0].size) {
|
|
752
|
+
throw new Error("the last multipart part cannot be larger than the others");
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
const deleteMultipart = (store: Database, bucket: string, key: string, uploadId: string): void => {
|
|
758
|
+
for (const part of multipartParts(store, bucket, key, uploadId)) {
|
|
759
|
+
r2DeletePart(store, bucket, key, uploadId, part.part_number);
|
|
760
|
+
}
|
|
761
|
+
store.query("DELETE FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
762
|
+
store.query("DELETE FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
const cleanupExpiredMultipart = (store: Database): void => {
|
|
766
|
+
const cutoff = new Date(Date.now() - MULTIPART_EXPIRY_MS).toISOString();
|
|
767
|
+
const expired = store
|
|
768
|
+
.query<R2UploadIdentity, [string]>("SELECT bucket, key, upload_id FROM r2_upload WHERE uploaded < ?")
|
|
769
|
+
.all(cutoff);
|
|
770
|
+
for (const upload of expired) deleteMultipart(store, upload.bucket, upload.key, upload.upload_id);
|
|
771
|
+
};
|
|
772
|
+
|
|
773
|
+
const commitR2Metadata = (
|
|
774
|
+
store: Database,
|
|
775
|
+
bucket: string,
|
|
776
|
+
row: R2Row,
|
|
777
|
+
previous: Pick<R2Row, "blob_id"> | null,
|
|
778
|
+
): void => {
|
|
779
|
+
try {
|
|
780
|
+
store
|
|
781
|
+
.query(
|
|
782
|
+
"INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json, blob_id) " +
|
|
783
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
|
|
784
|
+
"ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, " +
|
|
785
|
+
"custom_json=?7, blob_id=?8",
|
|
786
|
+
)
|
|
787
|
+
.run(bucket, row.key, row.size, row.etag, row.uploaded, row.http_json, row.custom_json, row.blob_id);
|
|
788
|
+
} catch (error) {
|
|
789
|
+
if (previous?.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, row.blob_id);
|
|
790
|
+
throw error;
|
|
791
|
+
}
|
|
792
|
+
if (previous && previous.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, previous.blob_id);
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
const putR2 = (
|
|
796
|
+
store: Database,
|
|
797
|
+
bucket: string,
|
|
798
|
+
key: string,
|
|
799
|
+
bytes: Uint8Array,
|
|
800
|
+
httpJson: string,
|
|
801
|
+
customJson: string,
|
|
802
|
+
): R2Row => {
|
|
803
|
+
const previous = store
|
|
804
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
805
|
+
.get(bucket, key);
|
|
806
|
+
const etag = createHash("sha256").update(bytes).digest("hex");
|
|
807
|
+
const row: R2Row = {
|
|
808
|
+
key,
|
|
809
|
+
size: bytes.byteLength,
|
|
810
|
+
etag,
|
|
811
|
+
uploaded: new Date().toISOString(),
|
|
812
|
+
http_json: httpJson,
|
|
813
|
+
custom_json: customJson,
|
|
814
|
+
blob_id: etag,
|
|
815
|
+
};
|
|
816
|
+
r2WriteBlob(store, bucket, key, bytes, etag);
|
|
817
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
818
|
+
return row;
|
|
819
|
+
};
|
|
820
|
+
|
|
821
|
+
const transferLimit = (): number => {
|
|
822
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
823
|
+
return Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
824
|
+
};
|
|
825
|
+
let activeUploads = 0;
|
|
826
|
+
const maxActiveUploads = 4;
|
|
827
|
+
|
|
828
|
+
const cleanupTransferTemps = (store: Database): void => {
|
|
829
|
+
const directory = r2BlobDir(store);
|
|
830
|
+
if (!directory || !existsSync(directory)) return;
|
|
831
|
+
const now = Date.now();
|
|
832
|
+
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
|
833
|
+
for (const name of readdirSync(directory)) {
|
|
834
|
+
if (!/^[0-9a-f]{24}\.upload\.tmp$/.test(name)) continue;
|
|
835
|
+
const path = join(directory, name);
|
|
836
|
+
try {
|
|
837
|
+
const tokenValue = name.slice(0, 24);
|
|
838
|
+
const ticket = store.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?").get(tokenValue);
|
|
839
|
+
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
840
|
+
} catch (error) {
|
|
841
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
|
|
846
|
+
const transferPath = (binding: string, tokenValue: string): string =>
|
|
847
|
+
`/__sb/r2/transfer/${encodeURIComponent(binding)}/${tokenValue}`;
|
|
420
848
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
849
|
+
const parseRange = (header: string | null, size: number): { start: number; end: number } | null => {
|
|
850
|
+
if (!header) return { start: 0, end: Math.max(0, size - 1) };
|
|
851
|
+
const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
852
|
+
if (!match || size === 0) return null;
|
|
853
|
+
if (!match[1]) {
|
|
854
|
+
const suffix = Number(match[2]);
|
|
855
|
+
if (!Number.isSafeInteger(suffix) || suffix <= 0) return null;
|
|
856
|
+
return { start: Math.max(0, size - suffix), end: size - 1 };
|
|
857
|
+
}
|
|
858
|
+
const start = Number(match[1]);
|
|
859
|
+
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
860
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return null;
|
|
861
|
+
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
862
|
+
};
|
|
863
|
+
|
|
864
|
+
const transfer = async (request: Request): Promise<Response> => {
|
|
865
|
+
const url = new URL(request.url);
|
|
866
|
+
const match = /^\/__sb\/r2\/transfer\/([A-Z][A-Z0-9_]*)\/([0-9a-f]{24})$/.exec(url.pathname);
|
|
867
|
+
if (!match) return new Response("not found", { status: 404 });
|
|
868
|
+
let store: Database;
|
|
869
|
+
let bucket: string;
|
|
870
|
+
try {
|
|
871
|
+
({ store, part: bucket } = storeFor("r2", requireR2(match[1])));
|
|
872
|
+
} catch {
|
|
873
|
+
return new Response("not found", { status: 404 });
|
|
874
|
+
}
|
|
875
|
+
const tokenValue = match[2];
|
|
876
|
+
const ticket = store
|
|
877
|
+
.query<R2TransferRow, [string]>(
|
|
878
|
+
"SELECT bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json, used " +
|
|
879
|
+
"FROM r2_transfer WHERE token = ?",
|
|
880
|
+
)
|
|
881
|
+
.get(tokenValue);
|
|
882
|
+
const method = request.method.toUpperCase();
|
|
883
|
+
const allowedMethod = ticket?.method === "upload" ? "PUT" : ticket?.method === "download" ? "GET, HEAD" : "";
|
|
884
|
+
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
885
|
+
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
886
|
+
}
|
|
887
|
+
if ((ticket.method === "upload" && method !== "PUT") || (ticket.method === "download" && method !== "GET" && method !== "HEAD")) {
|
|
888
|
+
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
889
|
+
}
|
|
890
|
+
if (ticket.method === "download") {
|
|
891
|
+
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, ticket.key);
|
|
892
|
+
if (!row) return new Response("not found", { status: 404 });
|
|
893
|
+
const etag = `"${row.etag}"`;
|
|
894
|
+
if (request.headers.get("if-none-match") === etag) return new Response(null, { status: 304, headers: { etag } });
|
|
895
|
+
const rangeHeader = request.headers.get("range");
|
|
896
|
+
const range = parseRange(rangeHeader, row.size);
|
|
897
|
+
if (!range) {
|
|
898
|
+
return new Response(null, { status: 416, headers: { "content-range": `bytes */${row.size}`, etag } });
|
|
899
|
+
}
|
|
900
|
+
const path = r2BlobPath(store, bucket, ticket.key, row.blob_id);
|
|
901
|
+
if (!path || !existsSync(path)) return new Response("not found", { status: 404 });
|
|
902
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
903
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
904
|
+
const length = row.size === 0 ? 0 : range.end - range.start + 1;
|
|
905
|
+
const metadata = jsonObject(parseJsonValue(row.http_json));
|
|
906
|
+
const headers = new Headers({
|
|
907
|
+
"accept-ranges": "bytes",
|
|
908
|
+
"content-length": String(length),
|
|
909
|
+
etag,
|
|
910
|
+
});
|
|
911
|
+
if (isString(metadata?.contentType)) headers.set("content-type", metadata.contentType);
|
|
912
|
+
if (rangeHeader) headers.set("content-range", `bytes ${range.start}-${range.end}/${row.size}`);
|
|
913
|
+
const body = method === "HEAD" ? null : Bun.file(path).slice(range.start, range.start + length);
|
|
914
|
+
return new Response(body, { status: rangeHeader ? 206 : 200, headers });
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
const declared = request.headers.has("content-length") ? Number(request.headers.get("content-length")) : NaN;
|
|
918
|
+
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
919
|
+
return new Response("invalid content length", { status: 400 });
|
|
920
|
+
}
|
|
921
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes) return new Response("upload too large", { status: 413 });
|
|
922
|
+
const directory = r2BlobDir(store);
|
|
923
|
+
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
924
|
+
if (activeUploads >= maxActiveUploads) {
|
|
925
|
+
return new Response("too many active uploads", { status: 503, headers: { "retry-after": "5" } });
|
|
926
|
+
}
|
|
927
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
928
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
929
|
+
activeUploads++;
|
|
930
|
+
const temporary = join(directory, `${tokenValue}.upload.tmp`);
|
|
931
|
+
await (async () => {
|
|
932
|
+
mkdirSync(directory, { recursive: true });
|
|
933
|
+
const reservation = await open(temporary, "wx");
|
|
934
|
+
await reservation.close();
|
|
935
|
+
})().catch((error) => {
|
|
936
|
+
activeUploads--;
|
|
937
|
+
throw error;
|
|
938
|
+
});
|
|
939
|
+
const writer = Bun.file(temporary).writer({ highWaterMark: 64 * 1024 });
|
|
940
|
+
const hash = createHash("sha256");
|
|
941
|
+
let size = 0;
|
|
942
|
+
try {
|
|
943
|
+
if (request.body) {
|
|
944
|
+
for await (const chunk of request.body) {
|
|
945
|
+
size += chunk.byteLength;
|
|
946
|
+
if (size > ticket.max_bytes) throw new RangeError("upload too large");
|
|
947
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
948
|
+
if (floor > 0 && availableBytes() < chunk.byteLength + floor) {
|
|
949
|
+
const error = new Error("R2 storage is temporarily unavailable");
|
|
950
|
+
Object.assign(error, { code: "SB_R2_DISK_FLOOR" });
|
|
951
|
+
throw error;
|
|
952
|
+
}
|
|
953
|
+
hash.update(chunk);
|
|
954
|
+
const written = await writer.write(chunk);
|
|
955
|
+
if (written !== chunk.byteLength) throw new Error("upload write was incomplete");
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
await writer.end();
|
|
959
|
+
const persisted = await open(temporary, "r");
|
|
960
|
+
try {
|
|
961
|
+
await persisted.sync();
|
|
962
|
+
} finally {
|
|
963
|
+
await persisted.close();
|
|
964
|
+
}
|
|
965
|
+
const etag = hash.digest("hex");
|
|
966
|
+
if (ticket.expected_sha256 && ticket.expected_sha256 !== etag) {
|
|
967
|
+
rmSync(temporary);
|
|
968
|
+
settleR2(tokenValue);
|
|
969
|
+
return Response.json({ error: "sha256 mismatch", actual: etag }, { status: 422 });
|
|
970
|
+
}
|
|
971
|
+
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
972
|
+
renameSync(temporary, destination);
|
|
973
|
+
const previous = store
|
|
974
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>("SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?")
|
|
975
|
+
.get(bucket, ticket.key);
|
|
976
|
+
const row: R2Row = {
|
|
977
|
+
key: ticket.key,
|
|
978
|
+
size,
|
|
979
|
+
etag,
|
|
980
|
+
uploaded: new Date().toISOString(),
|
|
981
|
+
http_json: ticket.http_json,
|
|
982
|
+
custom_json: ticket.custom_json,
|
|
983
|
+
blob_id: etag,
|
|
984
|
+
};
|
|
985
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
986
|
+
settleR2(tokenValue);
|
|
987
|
+
return Response.json({ object: r2Row(row) }, { status: 201, headers: { etag: `"${etag}"` } });
|
|
988
|
+
} catch (error) {
|
|
989
|
+
try {
|
|
990
|
+
await writer.end();
|
|
991
|
+
} catch {
|
|
992
|
+
// Preserve the original upload failure while releasing the file sink.
|
|
993
|
+
}
|
|
994
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
995
|
+
settleR2(tokenValue);
|
|
996
|
+
if (error instanceof RangeError) {
|
|
997
|
+
recordR2TransferMetric("rejected_size");
|
|
998
|
+
return new Response(error.message, { status: 413 });
|
|
999
|
+
}
|
|
1000
|
+
if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "SB_R2_DISK_FLOOR")) {
|
|
1001
|
+
recordR2TransferMetric("rejected_disk");
|
|
1002
|
+
return new Response("R2 storage is temporarily unavailable", { status: 507 });
|
|
1003
|
+
}
|
|
1004
|
+
throw error;
|
|
1005
|
+
} finally {
|
|
1006
|
+
activeUploads--;
|
|
1007
|
+
}
|
|
1008
|
+
};
|
|
1009
|
+
|
|
1010
|
+
const completeMultipart = (
|
|
1011
|
+
store: Database,
|
|
1012
|
+
bucket: string,
|
|
1013
|
+
key: string,
|
|
1014
|
+
uploadId: string,
|
|
1015
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
1016
|
+
): ReturnType<typeof r2Row> => {
|
|
1017
|
+
const upload = requireMultipart(store, bucket, key, uploadId);
|
|
1018
|
+
const parts = multipartParts(store, bucket, key, uploadId);
|
|
1019
|
+
validateCompletedParts(parts, requested);
|
|
1020
|
+
|
|
1021
|
+
const hash = createHash("sha256");
|
|
1022
|
+
let size = 0;
|
|
1023
|
+
const blobDirectory = r2BlobDir(store);
|
|
1024
|
+
let inMemoryBody: Uint8Array | null = null;
|
|
1025
|
+
let temporary: string | null = null;
|
|
1026
|
+
if (blobDirectory) {
|
|
1027
|
+
mkdirSync(blobDirectory, { recursive: true });
|
|
1028
|
+
temporary = join(blobDirectory, `${uploadId}.tmp`);
|
|
1029
|
+
writeFileSync(temporary, new Uint8Array(0));
|
|
1030
|
+
try {
|
|
1031
|
+
for (const part of parts) {
|
|
1032
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1033
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1034
|
+
appendFileSync(temporary, bytes);
|
|
1035
|
+
hash.update(bytes);
|
|
1036
|
+
size += bytes.byteLength;
|
|
1037
|
+
}
|
|
1038
|
+
} catch (error) {
|
|
1039
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
1040
|
+
throw error;
|
|
1041
|
+
}
|
|
1042
|
+
} else {
|
|
1043
|
+
const chunks: Uint8Array[] = [];
|
|
1044
|
+
for (const part of parts) {
|
|
1045
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1046
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1047
|
+
chunks.push(bytes);
|
|
1048
|
+
hash.update(bytes);
|
|
1049
|
+
size += bytes.byteLength;
|
|
1050
|
+
}
|
|
1051
|
+
inMemoryBody = Buffer.concat(chunks);
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
const etag = hash.digest("hex");
|
|
1055
|
+
const previous = store
|
|
1056
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1057
|
+
.get(bucket, key);
|
|
1058
|
+
if (temporary) renameSync(temporary, r2BlobPath(store, bucket, key, etag)!);
|
|
1059
|
+
else r2WriteBlob(store, bucket, key, inMemoryBody ?? new Uint8Array(0), etag);
|
|
1060
|
+
const row: R2Row = {
|
|
1061
|
+
key,
|
|
1062
|
+
size,
|
|
1063
|
+
etag,
|
|
1064
|
+
uploaded: new Date().toISOString(),
|
|
1065
|
+
http_json: upload.http_json,
|
|
1066
|
+
custom_json: upload.custom_json,
|
|
1067
|
+
blob_id: etag,
|
|
1068
|
+
};
|
|
1069
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
1070
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1071
|
+
return r2Row(row);
|
|
1072
|
+
};
|
|
426
1073
|
|
|
427
1074
|
const r2Row = (r: R2Row) => ({
|
|
428
1075
|
key: r.key,
|
|
@@ -617,40 +1264,129 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
617
1264
|
|
|
618
1265
|
case "r2.put": {
|
|
619
1266
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
620
|
-
const
|
|
621
|
-
const
|
|
1267
|
+
const key = str(msg.key);
|
|
1268
|
+
const bytes = new TextEncoder().encode(str(msg.body));
|
|
1269
|
+
return {
|
|
1270
|
+
ok: true,
|
|
1271
|
+
object: r2Row(
|
|
1272
|
+
putR2(
|
|
1273
|
+
store,
|
|
1274
|
+
bucket,
|
|
1275
|
+
key,
|
|
1276
|
+
bytes,
|
|
1277
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1278
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1279
|
+
),
|
|
1280
|
+
),
|
|
1281
|
+
};
|
|
1282
|
+
}
|
|
1283
|
+
case "r2.multipart.create": {
|
|
1284
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1285
|
+
cleanupExpiredMultipart(store);
|
|
1286
|
+
const key = str(msg.key);
|
|
1287
|
+
const uploadId = str(msg.uploadId) || newId();
|
|
1288
|
+
const uploaded = new Date().toISOString();
|
|
622
1289
|
store
|
|
623
1290
|
.query(
|
|
624
|
-
"INSERT INTO
|
|
625
|
-
"ON CONFLICT (bucket, key) DO
|
|
1291
|
+
"INSERT INTO r2_upload (bucket, key, upload_id, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1292
|
+
"ON CONFLICT (bucket, key, upload_id) DO NOTHING",
|
|
626
1293
|
)
|
|
627
1294
|
.run(
|
|
628
1295
|
bucket,
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
etag,
|
|
633
|
-
new Date().toISOString(),
|
|
1296
|
+
key,
|
|
1297
|
+
uploadId,
|
|
1298
|
+
uploaded,
|
|
634
1299
|
JSON.stringify(msg.httpMetadata ?? {}),
|
|
635
1300
|
JSON.stringify(msg.customMetadata ?? {}),
|
|
636
1301
|
);
|
|
637
|
-
return {
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
1302
|
+
return { ok: true, key, uploadId };
|
|
1303
|
+
}
|
|
1304
|
+
case "r2.transfer.create": {
|
|
1305
|
+
const binding = requireR2(msg.bucket);
|
|
1306
|
+
const { store, part: bucket } = storeFor("r2", binding);
|
|
1307
|
+
const key = str(msg.key);
|
|
1308
|
+
const method = msg.method === "download" ? "download" : msg.method === "upload" ? "upload" : "";
|
|
1309
|
+
if (!method) throw new Error("transfer method must be upload or download");
|
|
1310
|
+
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1311
|
+
const expires = Date.now() + expiresIn * 1000;
|
|
1312
|
+
const maxBytes =
|
|
1313
|
+
method === "upload"
|
|
1314
|
+
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1315
|
+
: 0;
|
|
1316
|
+
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1317
|
+
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1318
|
+
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
1319
|
+
}
|
|
1320
|
+
store.query("DELETE FROM r2_transfer WHERE expires < ? OR used = 1").run(Date.now());
|
|
1321
|
+
cleanupTransferTemps(store);
|
|
1322
|
+
const tokenValue = randomBytes(12).toString("hex");
|
|
1323
|
+
const reservation = method === "upload" ? reserveR2(tokenValue, maxBytes, expires) : "ok";
|
|
1324
|
+
if (reservation === "quota") {
|
|
1325
|
+
recordR2TransferMetric("rejected_quota");
|
|
1326
|
+
throw new Error("R2 account quota would be exceeded");
|
|
1327
|
+
}
|
|
1328
|
+
if (reservation === "disk") {
|
|
1329
|
+
recordR2TransferMetric("rejected_disk");
|
|
1330
|
+
throw new Error("R2 storage is temporarily unavailable");
|
|
1331
|
+
}
|
|
1332
|
+
try {
|
|
1333
|
+
store
|
|
1334
|
+
.query(
|
|
1335
|
+
"INSERT INTO r2_transfer (token, bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json) " +
|
|
1336
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
|
|
1337
|
+
)
|
|
1338
|
+
.run(
|
|
1339
|
+
tokenValue,
|
|
1340
|
+
bucket,
|
|
1341
|
+
key,
|
|
1342
|
+
method,
|
|
1343
|
+
expires,
|
|
1344
|
+
maxBytes,
|
|
1345
|
+
expectedSha256,
|
|
1346
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1347
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1348
|
+
);
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
if (reservation === "ok" && method === "upload") settleR2(tokenValue);
|
|
1351
|
+
throw error;
|
|
1352
|
+
}
|
|
1353
|
+
recordR2TransferMetric("tickets_issued");
|
|
1354
|
+
return { ok: true, url: transferPath(binding, tokenValue), expiresAt: new Date(expires).toISOString() };
|
|
1355
|
+
}
|
|
1356
|
+
case "r2.multipart.complete": {
|
|
1357
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1358
|
+
const key = str(msg.key);
|
|
1359
|
+
const uploadId = str(msg.uploadId);
|
|
1360
|
+
const object = completeMultipart(store, bucket, key, uploadId, requestedParts(msg.parts));
|
|
1361
|
+
return { ok: true, object };
|
|
1362
|
+
}
|
|
1363
|
+
case "r2.multipart.abort": {
|
|
1364
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1365
|
+
const key = str(msg.key);
|
|
1366
|
+
const uploadId = str(msg.uploadId);
|
|
1367
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
1368
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1369
|
+
return { ok: true };
|
|
641
1370
|
}
|
|
642
1371
|
case "r2.get":
|
|
643
1372
|
case "r2.head": {
|
|
644
1373
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
645
|
-
const
|
|
646
|
-
|
|
647
|
-
.get(bucket, str(msg.key));
|
|
1374
|
+
const key = str(msg.key);
|
|
1375
|
+
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
648
1376
|
if (!row) return { ok: true, found: false };
|
|
649
|
-
|
|
1377
|
+
const body = msg.op === "r2.get"
|
|
1378
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1379
|
+
: undefined;
|
|
1380
|
+
return { ok: true, found: true, object: r2Row(row), body };
|
|
650
1381
|
}
|
|
651
1382
|
case "r2.delete": {
|
|
652
1383
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
653
|
-
|
|
1384
|
+
const key = str(msg.key);
|
|
1385
|
+
const row = store
|
|
1386
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1387
|
+
.get(bucket, key);
|
|
1388
|
+
store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
|
|
1389
|
+
if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
|
|
654
1390
|
return { ok: true };
|
|
655
1391
|
}
|
|
656
1392
|
case "r2.list": {
|
|
@@ -832,38 +1568,51 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
832
1568
|
*/
|
|
833
1569
|
async function dispatchBinary(msg: Frame, binary: Uint8Array): Promise<{ reply: Frame; bytes?: Uint8Array }> {
|
|
834
1570
|
if (msg.op === "r2.put") {
|
|
835
|
-
// The body arrived as bytes;
|
|
836
|
-
//
|
|
1571
|
+
// The body arrived as bytes; write it straight to its own file rather
|
|
1572
|
+
// than into SQLite (#56) — no escape, no parse, no re-encode, and no
|
|
1573
|
+
// second full-object copy through the database's own page cache.
|
|
1574
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1575
|
+
const key = str(msg.key);
|
|
1576
|
+
const row = putR2(
|
|
1577
|
+
store,
|
|
1578
|
+
bucket,
|
|
1579
|
+
key,
|
|
1580
|
+
binary,
|
|
1581
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1582
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1583
|
+
);
|
|
1584
|
+
return { reply: { ok: true, object: r2Row(row) } };
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
if (msg.op === "r2.multipart.put") {
|
|
837
1588
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1589
|
+
const key = str(msg.key);
|
|
1590
|
+
const uploadId = str(msg.uploadId);
|
|
1591
|
+
const partNumber = Number(msg.partNumber);
|
|
1592
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
|
|
1593
|
+
throw new Error("partNumber must be an integer between 1 and 10000");
|
|
1594
|
+
}
|
|
1595
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
838
1596
|
const etag = createHash("sha256").update(binary).digest("hex");
|
|
839
|
-
|
|
1597
|
+
r2WritePart(store, bucket, key, uploadId, partNumber, binary);
|
|
840
1598
|
store
|
|
841
1599
|
.query(
|
|
842
|
-
"INSERT INTO
|
|
843
|
-
"ON CONFLICT (bucket, key) DO UPDATE SET
|
|
1600
|
+
"INSERT INTO r2_part (bucket, key, upload_id, part_number, size, etag) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1601
|
+
"ON CONFLICT (bucket, key, upload_id, part_number) DO UPDATE SET size=?5, etag=?6",
|
|
844
1602
|
)
|
|
845
|
-
.run(
|
|
846
|
-
|
|
847
|
-
str(msg.key),
|
|
848
|
-
binary,
|
|
849
|
-
binary.byteLength,
|
|
850
|
-
etag,
|
|
851
|
-
uploaded,
|
|
852
|
-
JSON.stringify(msg.httpMetadata ?? {}),
|
|
853
|
-
JSON.stringify(msg.customMetadata ?? {}),
|
|
854
|
-
);
|
|
855
|
-
return {
|
|
856
|
-
reply: { ok: true, object: { key: str(msg.key), size: binary.byteLength, etag, uploaded } },
|
|
857
|
-
};
|
|
1603
|
+
.run(bucket, key, uploadId, partNumber, binary.byteLength, etag);
|
|
1604
|
+
return { reply: { ok: true, part: { partNumber, etag } } };
|
|
858
1605
|
}
|
|
859
1606
|
|
|
860
1607
|
if (msg.op === "r2.get") {
|
|
861
1608
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
862
|
-
const
|
|
863
|
-
|
|
864
|
-
.get(bucket, str(msg.key));
|
|
1609
|
+
const key = str(msg.key);
|
|
1610
|
+
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
865
1611
|
if (!row) return { reply: { ok: true, found: false } };
|
|
866
|
-
return {
|
|
1612
|
+
return {
|
|
1613
|
+
reply: { ok: true, found: true, object: r2Row(row) },
|
|
1614
|
+
bytes: r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0),
|
|
1615
|
+
};
|
|
867
1616
|
}
|
|
868
1617
|
|
|
869
1618
|
if (msg.op === "assets.get") {
|
|
@@ -901,7 +1650,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
901
1650
|
if (!msg) throw new Error("request frame was not a JSON object");
|
|
902
1651
|
if (token && str(msg.token) !== token) return frameOf(encodeV1({ ok: false, error: "unauthorized" }));
|
|
903
1652
|
const binary = new Uint8Array(payload.subarray(5 + jsonLen));
|
|
1653
|
+
const id = isSafeInteger(msg.id) ? msg.id : null;
|
|
1654
|
+
const replayKey = id === null ? null : `${str(msg.op)}:${id}`;
|
|
1655
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) {
|
|
1656
|
+
const seen = replayed.get(replayKey);
|
|
1657
|
+
if (seen) return frameOf(encodeV1(seen));
|
|
1658
|
+
}
|
|
904
1659
|
const { reply, bytes } = await dispatchBinary(msg, binary);
|
|
1660
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
|
|
905
1661
|
return frameOf(encodeV1(reply, bytes));
|
|
906
1662
|
} catch (e) {
|
|
907
1663
|
return frameOf(encodeV1({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
@@ -928,6 +1684,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
928
1684
|
"d1.backup",
|
|
929
1685
|
"ratelimit.check",
|
|
930
1686
|
"r2.put",
|
|
1687
|
+
"r2.multipart.create",
|
|
1688
|
+
"r2.multipart.put",
|
|
1689
|
+
"r2.multipart.complete",
|
|
1690
|
+
"r2.multipart.abort",
|
|
1691
|
+
"r2.transfer.create",
|
|
931
1692
|
"r2.delete",
|
|
932
1693
|
"do.storage.put",
|
|
933
1694
|
"do.storage.delete",
|
|
@@ -941,6 +1702,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
941
1702
|
* milliseconds, so this only has to outlive a reconnect, not a session. */
|
|
942
1703
|
const REPLAY_WINDOW = 256;
|
|
943
1704
|
|
|
1705
|
+
const rememberReplay = (key: string, reply: Frame): void => {
|
|
1706
|
+
replayed.set(key, reply);
|
|
1707
|
+
if (replayed.size > REPLAY_WINDOW) {
|
|
1708
|
+
const oldest = replayed.keys().next().value;
|
|
1709
|
+
if (oldest !== undefined) replayed.delete(oldest);
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
|
|
944
1713
|
/**
|
|
945
1714
|
* Run `msg`, or replay what it answered last time. Keyed by the id the sprout
|
|
946
1715
|
* put on the request; a request without one is always run.
|
|
@@ -953,12 +1722,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
953
1722
|
const seen = replayed.get(key);
|
|
954
1723
|
if (seen) return seen;
|
|
955
1724
|
const reply = await dispatch(msg);
|
|
956
|
-
|
|
957
|
-
// Insertion-ordered, so the oldest key is the first one out.
|
|
958
|
-
if (replayed.size > REPLAY_WINDOW) {
|
|
959
|
-
const oldest = replayed.keys().next().value;
|
|
960
|
-
if (oldest !== undefined) replayed.delete(oldest);
|
|
961
|
-
}
|
|
1725
|
+
rememberReplay(key, reply);
|
|
962
1726
|
return reply;
|
|
963
1727
|
}
|
|
964
1728
|
|
|
@@ -1105,6 +1869,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1105
1869
|
dispatch,
|
|
1106
1870
|
handlePayload,
|
|
1107
1871
|
handleFrame,
|
|
1872
|
+
transfer,
|
|
1108
1873
|
close: () => {
|
|
1109
1874
|
for (const t of timers) clearInterval(t);
|
|
1110
1875
|
for (const conn of d1Conns.values()) conn.close();
|
|
@@ -1192,17 +1957,33 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
1192
1957
|
},
|
|
1193
1958
|
},
|
|
1194
1959
|
});
|
|
1195
|
-
return { port: server.port, stop: () => server.stop(true) };
|
|
1960
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
/** Start the HTTP listener that serves short-lived direct R2 transfer tickets. */
|
|
1964
|
+
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
1965
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
1966
|
+
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
1967
|
+
const server = Bun.serve({ hostname, port, maxRequestBodySize, fetch(request, server) {
|
|
1968
|
+
server.timeout(request, 255);
|
|
1969
|
+
return broker.transfer(request);
|
|
1970
|
+
} });
|
|
1971
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1196
1972
|
}
|
|
1197
1973
|
|
|
1198
1974
|
if (import.meta.main) {
|
|
1199
1975
|
const { values } = parseArgs({
|
|
1200
1976
|
options: {
|
|
1201
1977
|
port: { type: "string" },
|
|
1978
|
+
"transfer-port": { type: "string" },
|
|
1202
1979
|
token: { type: "string" },
|
|
1203
1980
|
db: { type: "string" },
|
|
1204
1981
|
"data-dir": { type: "string" },
|
|
1205
1982
|
"resource-dir": { type: "string" },
|
|
1983
|
+
"owner-id": { type: "string" },
|
|
1984
|
+
"r2-quota-bytes": { type: "string" },
|
|
1985
|
+
"r2-resource-ids": { type: "string" },
|
|
1986
|
+
"r2-min-free-bytes": { type: "string" },
|
|
1206
1987
|
bindings: { type: "string" },
|
|
1207
1988
|
secrets: { type: "string" },
|
|
1208
1989
|
"sprout-url": { type: "string" },
|
|
@@ -1251,6 +2032,10 @@ if (import.meta.main) {
|
|
|
1251
2032
|
db: values.db,
|
|
1252
2033
|
dataDir: values["data-dir"],
|
|
1253
2034
|
resourceDir: values["resource-dir"],
|
|
2035
|
+
ownerId: values["owner-id"],
|
|
2036
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2037
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2038
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
1254
2039
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
1255
2040
|
bindings,
|
|
1256
2041
|
secrets,
|
|
@@ -1261,7 +2046,9 @@ if (import.meta.main) {
|
|
|
1261
2046
|
dispatchEnabled: () => dispatchEnabled,
|
|
1262
2047
|
});
|
|
1263
2048
|
const { port } = listen(broker, "127.0.0.1", Number(values.port ?? process.env.SB_BROKER_PORT ?? 0));
|
|
2049
|
+
const transferPort = Number(values["transfer-port"] ?? process.env.SB_R2_TRANSFER_PORT ?? 0);
|
|
2050
|
+
const transferServer = transferPort > 0 ? listenTransfers(broker, "127.0.0.1", transferPort) : null;
|
|
1264
2051
|
console.log(
|
|
1265
|
-
`sproutboat broker: 127.0.0.1:${port} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
2052
|
+
`sproutboat broker: 127.0.0.1:${port} transfer=${transferServer?.port ?? "off"} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
1266
2053
|
);
|
|
1267
2054
|
}
|