@sproutboat/wire 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/package.json +1 -1
- package/src/broker.test.ts +321 -0
- package/src/broker.ts +838 -61
package/src/broker.ts
CHANGED
|
@@ -18,8 +18,9 @@
|
|
|
18
18
|
* the only SSRF control today.
|
|
19
19
|
*/
|
|
20
20
|
import { Database, type Statement } from "bun:sqlite";
|
|
21
|
-
import { createHash } from "node:crypto";
|
|
22
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
21
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
22
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statfsSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { open } from "node:fs/promises";
|
|
23
24
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
24
25
|
import { parseArgs } from "node:util";
|
|
25
26
|
import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
|
|
@@ -84,6 +85,16 @@ export type BrokerOptions = {
|
|
|
84
85
|
* data outlives a redeploy.
|
|
85
86
|
*/
|
|
86
87
|
resourceDir?: string;
|
|
88
|
+
/** Account whose R2 resources this broker may mutate. */
|
|
89
|
+
ownerId?: string;
|
|
90
|
+
/** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
|
|
91
|
+
r2QuotaBytes?: number;
|
|
92
|
+
/** Resource ids belonging to ownerId, used to seed the shared R2 usage ledger. */
|
|
93
|
+
r2ResourceIds?: string[];
|
|
94
|
+
/** Free disk capacity retained while direct R2 uploads are accepted. */
|
|
95
|
+
r2MinFreeBytes?: number;
|
|
96
|
+
/** Test hook for checking the filesystem capacity available to temporary uploads. */
|
|
97
|
+
freeBytes?: (path: string) => number;
|
|
87
98
|
token?: string;
|
|
88
99
|
bindings?: Partial<Bindings>;
|
|
89
100
|
secrets?: Record<string, string>;
|
|
@@ -115,7 +126,28 @@ type R2Row = {
|
|
|
115
126
|
uploaded: string;
|
|
116
127
|
http_json: string;
|
|
117
128
|
custom_json: string;
|
|
129
|
+
blob_id: string | null;
|
|
118
130
|
};
|
|
131
|
+
type R2UploadRow = {
|
|
132
|
+
key: string;
|
|
133
|
+
uploaded: string;
|
|
134
|
+
http_json: string;
|
|
135
|
+
custom_json: string;
|
|
136
|
+
};
|
|
137
|
+
type R2PartRow = { part_number: number; size: number; etag: string };
|
|
138
|
+
type R2UploadIdentity = { bucket: string; key: string; upload_id: string };
|
|
139
|
+
type R2TransferRow = {
|
|
140
|
+
bucket: string;
|
|
141
|
+
key: string;
|
|
142
|
+
method: string;
|
|
143
|
+
expires: number;
|
|
144
|
+
max_bytes: number;
|
|
145
|
+
expected_sha256: string;
|
|
146
|
+
http_json: string;
|
|
147
|
+
custom_json: string;
|
|
148
|
+
used: number;
|
|
149
|
+
};
|
|
150
|
+
const MULTIPART_EXPIRY_MS = 7 * 24 * 60 * 60 * 1000;
|
|
119
151
|
const isNumber = (v: JsonValue | undefined): v is number => Number.isFinite(v);
|
|
120
152
|
const sqlParams = (v: JsonValue | undefined): SqlParam[] => {
|
|
121
153
|
if (!Array.isArray(v)) return [];
|
|
@@ -144,6 +176,8 @@ export type Broker = {
|
|
|
144
176
|
* negotiation.
|
|
145
177
|
*/
|
|
146
178
|
handleFrame(payload: Buffer): Promise<Buffer>;
|
|
179
|
+
/** Serve one reserved direct R2 upload or download request. */
|
|
180
|
+
transfer(request: Request): Promise<Response>;
|
|
147
181
|
close(): void;
|
|
148
182
|
};
|
|
149
183
|
|
|
@@ -230,8 +264,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
230
264
|
conn.exec("PRAGMA journal_mode = WAL");
|
|
231
265
|
conn.exec("PRAGMA synchronous = NORMAL");
|
|
232
266
|
conn.exec(
|
|
233
|
-
"CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL,
|
|
267
|
+
"CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, " +
|
|
268
|
+
"expires_at INTEGER, PRIMARY KEY (ns, key))",
|
|
234
269
|
);
|
|
270
|
+
const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
|
|
271
|
+
if (!kvColumns.some((column) => column.name === "expires_at")) conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
|
|
235
272
|
// #56 — no `body` column: an object's bytes live in their own file under
|
|
236
273
|
// `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
|
|
237
274
|
// Doubling every object through SQLite's own page cache on top of the
|
|
@@ -241,8 +278,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
241
278
|
conn.exec(
|
|
242
279
|
"CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
|
|
243
280
|
"etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
281
|
+
"blob_id TEXT, " +
|
|
244
282
|
"PRIMARY KEY (bucket, key))",
|
|
245
283
|
);
|
|
284
|
+
const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
|
|
285
|
+
if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
|
|
286
|
+
conn.exec(
|
|
287
|
+
"CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
288
|
+
"uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
289
|
+
"PRIMARY KEY (bucket, key, upload_id))",
|
|
290
|
+
);
|
|
291
|
+
conn.exec(
|
|
292
|
+
"CREATE TABLE IF NOT EXISTS r2_part (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
293
|
+
"part_number INTEGER NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, " +
|
|
294
|
+
"PRIMARY KEY (bucket, key, upload_id, part_number))",
|
|
295
|
+
);
|
|
296
|
+
conn.exec(
|
|
297
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer (token TEXT PRIMARY KEY, bucket TEXT NOT NULL, key TEXT NOT NULL, " +
|
|
298
|
+
"method TEXT NOT NULL, expires INTEGER NOT NULL, max_bytes INTEGER NOT NULL, expected_sha256 TEXT NOT NULL DEFAULT '', " +
|
|
299
|
+
"http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', used INTEGER NOT NULL DEFAULT 0)",
|
|
300
|
+
);
|
|
246
301
|
conn.exec(
|
|
247
302
|
"CREATE TABLE IF NOT EXISTS mq (queue TEXT NOT NULL, id TEXT PRIMARY KEY, body TEXT NOT NULL, " +
|
|
248
303
|
"visible_at INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, dead INTEGER NOT NULL DEFAULT 0)",
|
|
@@ -290,6 +345,121 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
290
345
|
return conn;
|
|
291
346
|
};
|
|
292
347
|
|
|
348
|
+
const quotaOwner = opts.ownerId;
|
|
349
|
+
const quotaBytes = opts.r2QuotaBytes && opts.r2QuotaBytes > 0 ? opts.r2QuotaBytes : null;
|
|
350
|
+
const quotaDirectory = resourceDir;
|
|
351
|
+
if (quotaOwner && quotaDirectory && quotaBytes) mkdirSync(quotaDirectory, { recursive: true });
|
|
352
|
+
// One ledger for every account, shared by the brokers of all its deployments.
|
|
353
|
+
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
354
|
+
// requests cannot each independently decide that the same remaining bytes fit.
|
|
355
|
+
const quotaDb = quotaOwner && quotaDirectory && quotaBytes
|
|
356
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
357
|
+
: null;
|
|
358
|
+
if (quotaDb) {
|
|
359
|
+
const owner = quotaOwner!;
|
|
360
|
+
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
361
|
+
quotaDb.exec(
|
|
362
|
+
"CREATE TABLE IF NOT EXISTS r2_account_quota (owner TEXT PRIMARY KEY, used_bytes INTEGER NOT NULL, reserved_bytes INTEGER NOT NULL)",
|
|
363
|
+
);
|
|
364
|
+
quotaDb.exec(
|
|
365
|
+
"CREATE TABLE IF NOT EXISTS r2_quota_reservation (token TEXT PRIMARY KEY, owner TEXT NOT NULL, bytes INTEGER NOT NULL, expires INTEGER NOT NULL)",
|
|
366
|
+
);
|
|
367
|
+
quotaDb.exec(
|
|
368
|
+
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
369
|
+
);
|
|
370
|
+
const existing = quotaDb.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?").get(owner);
|
|
371
|
+
if (!existing) {
|
|
372
|
+
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
373
|
+
}
|
|
374
|
+
quotaDb.query("INSERT OR IGNORE INTO r2_transfer_metric (owner) VALUES (?)").run(owner);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const committedR2Bytes = (): number => {
|
|
378
|
+
if (!quotaDirectory) return 0;
|
|
379
|
+
let used = 0;
|
|
380
|
+
for (const id of opts.r2ResourceIds ?? []) {
|
|
381
|
+
// Open through resourceDb so older resource files gain the multipart
|
|
382
|
+
// tables before the aggregate is queried.
|
|
383
|
+
const store = resourceDb(id);
|
|
384
|
+
used += store.query<{ bytes: number }, []>(
|
|
385
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
386
|
+
).get()?.bytes ?? 0;
|
|
387
|
+
}
|
|
388
|
+
return used;
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
const availableBytes = (): number => {
|
|
392
|
+
if (!quotaDirectory) return Number.POSITIVE_INFINITY;
|
|
393
|
+
if (opts.freeBytes) return opts.freeBytes(quotaDirectory);
|
|
394
|
+
const stats = statfsSync(quotaDirectory);
|
|
395
|
+
return Number(stats.bavail) * Number(stats.bsize);
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
399
|
+
if (!quotaDb || !quotaOwner) return;
|
|
400
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
401
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
402
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
403
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
404
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
405
|
+
};
|
|
406
|
+
|
|
407
|
+
const reserveR2 = (token: string, bytes: number, expires: number): "ok" | "quota" | "disk" => {
|
|
408
|
+
if (!quotaDb || !quotaOwner || !quotaBytes) return "ok";
|
|
409
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
410
|
+
try {
|
|
411
|
+
releaseExpiredR2Reservations();
|
|
412
|
+
const used = committedR2Bytes();
|
|
413
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
414
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
415
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
416
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
417
|
+
if (used + reserved + bytes > quotaBytes) {
|
|
418
|
+
quotaDb.exec("ROLLBACK");
|
|
419
|
+
return "quota";
|
|
420
|
+
}
|
|
421
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
422
|
+
if (availableBytes() < bytes + floor) {
|
|
423
|
+
quotaDb.exec("ROLLBACK");
|
|
424
|
+
return "disk";
|
|
425
|
+
}
|
|
426
|
+
quotaDb.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)").run(token, quotaOwner, bytes, expires);
|
|
427
|
+
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved + bytes, quotaOwner);
|
|
428
|
+
quotaDb.exec("COMMIT");
|
|
429
|
+
return "ok";
|
|
430
|
+
} catch (error) {
|
|
431
|
+
quotaDb.exec("ROLLBACK");
|
|
432
|
+
throw error;
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
const settleR2 = (token: string): void => {
|
|
437
|
+
if (!quotaDb || !quotaOwner) return;
|
|
438
|
+
quotaDb.exec("BEGIN IMMEDIATE");
|
|
439
|
+
try {
|
|
440
|
+
const reservation = quotaDb.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?").get(token);
|
|
441
|
+
if (!reservation) {
|
|
442
|
+
quotaDb.exec("COMMIT");
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
446
|
+
const used = committedR2Bytes();
|
|
447
|
+
const reserved = quotaDb.query<{ bytes: number }, [string]>(
|
|
448
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
449
|
+
).get(quotaOwner)?.bytes ?? 0;
|
|
450
|
+
quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
|
|
451
|
+
quotaDb.exec("COMMIT");
|
|
452
|
+
} catch (error) {
|
|
453
|
+
quotaDb.exec("ROLLBACK");
|
|
454
|
+
throw error;
|
|
455
|
+
}
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
const recordR2TransferMetric = (field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size"): void => {
|
|
459
|
+
if (!quotaDb || !quotaOwner) return;
|
|
460
|
+
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
461
|
+
};
|
|
462
|
+
|
|
293
463
|
/**
|
|
294
464
|
* Where a KV / R2 / queue binding stores: its resource's own file keyed by
|
|
295
465
|
* `id` when the config bound it to one, else the per-broker `db` partitioned
|
|
@@ -303,29 +473,48 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
303
473
|
};
|
|
304
474
|
|
|
305
475
|
type KvStmts = {
|
|
306
|
-
get: Statement<{ value: string }, [string, string]>;
|
|
307
|
-
put: Statement<unknown, [string, string, string]>;
|
|
476
|
+
get: Statement<{ value: string; expires_at: number | null }, [string, string]>;
|
|
477
|
+
put: Statement<unknown, [string, string, string, number | null]>;
|
|
308
478
|
del: Statement<unknown, [string, string]>;
|
|
309
|
-
list: Statement<{ key: string }, [string, string]>;
|
|
479
|
+
list: Statement<{ key: string }, [string, string, number]>;
|
|
310
480
|
};
|
|
311
481
|
const kvStmtCache = new Map<Database, KvStmts>();
|
|
312
482
|
const kvStmts = (store: Database): KvStmts => {
|
|
313
483
|
let stmts = kvStmtCache.get(store);
|
|
314
484
|
if (!stmts) {
|
|
315
485
|
stmts = {
|
|
316
|
-
get: store.query<{ value: string }, [string, string]>(
|
|
486
|
+
get: store.query<{ value: string; expires_at: number | null }, [string, string]>(
|
|
487
|
+
"SELECT value, expires_at FROM kv WHERE ns = ? AND key = ?",
|
|
488
|
+
),
|
|
317
489
|
put: store.query(
|
|
318
|
-
"INSERT INTO kv (ns, key, value) VALUES (?1, ?2, ?3
|
|
490
|
+
"INSERT INTO kv (ns, key, value, expires_at) VALUES (?1, ?2, ?3, ?4) " +
|
|
491
|
+
"ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4",
|
|
319
492
|
),
|
|
320
493
|
del: store.query("DELETE FROM kv WHERE ns = ? AND key = ?"),
|
|
321
|
-
|
|
322
|
-
|
|
494
|
+
// #58 — an expired row is filtered out here; a periodic sweep to hard-delete
|
|
495
|
+
// it is a separate concern (ponytail: rows accumulate until touched by a
|
|
496
|
+
// get/list/put on the same key/prefix, add a cron sweep if that matters).
|
|
497
|
+
list: store.query<{ key: string }, [string, string, number]>(
|
|
498
|
+
"SELECT key FROM kv WHERE ns = ? AND key LIKE ? || '%' AND (expires_at IS NULL OR expires_at > ?) ORDER BY key LIMIT 1000",
|
|
323
499
|
),
|
|
324
500
|
};
|
|
325
501
|
kvStmtCache.set(store, stmts);
|
|
326
502
|
}
|
|
327
503
|
return stmts;
|
|
328
504
|
};
|
|
505
|
+
/** CF's own floor. Cloudflare rejects `expirationTtl < 60`; matched here so a
|
|
506
|
+
* handler ported from Workers gets the same validation error either place. */
|
|
507
|
+
const MIN_KV_TTL_SECONDS = 60;
|
|
508
|
+
/** `{ expirationTtl }` (seconds from now) or `{ expiration }` (absolute unix
|
|
509
|
+
* seconds) -> ms epoch, or `null` for no expiry. */
|
|
510
|
+
const kvExpiresAt = (msg: Frame): number | null => {
|
|
511
|
+
if (isNumber(msg.expirationTtl)) {
|
|
512
|
+
if (msg.expirationTtl < MIN_KV_TTL_SECONDS) throw new Error(`expirationTtl must be at least ${MIN_KV_TTL_SECONDS} seconds`);
|
|
513
|
+
return Date.now() + msg.expirationTtl * 1000;
|
|
514
|
+
}
|
|
515
|
+
if (isNumber(msg.expiration)) return msg.expiration * 1000;
|
|
516
|
+
return null;
|
|
517
|
+
};
|
|
329
518
|
|
|
330
519
|
const bound =
|
|
331
520
|
(list: string[], kind: string) =>
|
|
@@ -428,39 +617,482 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
428
617
|
* `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
|
|
429
618
|
*/
|
|
430
619
|
const r2MemBlobs = new Map<string, Uint8Array>();
|
|
431
|
-
const
|
|
432
|
-
|
|
620
|
+
const r2MemParts = new Map<string, Uint8Array>();
|
|
621
|
+
const r2BlobId = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
622
|
+
createHash("sha256")
|
|
623
|
+
.update(generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`)
|
|
624
|
+
.digest("hex");
|
|
433
625
|
const r2BlobDir = (store: Database): string | null => {
|
|
434
626
|
const file = store.filename;
|
|
435
627
|
return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
|
|
436
628
|
};
|
|
437
|
-
const r2BlobPath = (store: Database, bucketPart: string, key: string): string | null => {
|
|
629
|
+
const r2BlobPath = (store: Database, bucketPart: string, key: string, generation?: string | null): string | null => {
|
|
438
630
|
const dir = r2BlobDir(store);
|
|
439
|
-
return dir ? join(dir, `${r2BlobId(bucketPart, key)}.blob`) : null;
|
|
631
|
+
return dir ? join(dir, `${r2BlobId(bucketPart, key, generation)}.blob`) : null;
|
|
632
|
+
};
|
|
633
|
+
const r2BlobMemKey = (bucketPart: string, key: string, generation?: string | null): string =>
|
|
634
|
+
generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`;
|
|
635
|
+
const r2WriteBlob = (
|
|
636
|
+
store: Database,
|
|
637
|
+
bucketPart: string,
|
|
638
|
+
key: string,
|
|
639
|
+
bytes: Uint8Array,
|
|
640
|
+
generation: string,
|
|
641
|
+
): void => {
|
|
642
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
643
|
+
if (!path) {
|
|
644
|
+
r2MemBlobs.set(r2BlobMemKey(bucketPart, key, generation), bytes);
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
648
|
+
const temporary = `${path}.tmp`;
|
|
649
|
+
writeFileSync(temporary, bytes);
|
|
650
|
+
renameSync(temporary, path);
|
|
651
|
+
};
|
|
652
|
+
const r2ReadBlob = (
|
|
653
|
+
store: Database,
|
|
654
|
+
bucketPart: string,
|
|
655
|
+
key: string,
|
|
656
|
+
generation?: string | null,
|
|
657
|
+
): Uint8Array | null => {
|
|
658
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
659
|
+
if (!path) return r2MemBlobs.get(r2BlobMemKey(bucketPart, key, generation)) ?? null;
|
|
660
|
+
return existsSync(path) ? readFileSync(path) : null;
|
|
440
661
|
};
|
|
441
|
-
const
|
|
442
|
-
const path = r2BlobPath(store, bucketPart, key);
|
|
662
|
+
const r2DeleteBlob = (store: Database, bucketPart: string, key: string, generation?: string | null): void => {
|
|
663
|
+
const path = r2BlobPath(store, bucketPart, key, generation);
|
|
443
664
|
if (!path) {
|
|
444
|
-
r2MemBlobs.
|
|
665
|
+
r2MemBlobs.delete(r2BlobMemKey(bucketPart, key, generation));
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
if (existsSync(path)) rmSync(path);
|
|
669
|
+
};
|
|
670
|
+
const r2PartId = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
671
|
+
createHash("sha256").update(`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`).digest("hex");
|
|
672
|
+
const r2PartPath = (
|
|
673
|
+
store: Database,
|
|
674
|
+
bucketPart: string,
|
|
675
|
+
key: string,
|
|
676
|
+
uploadId: string,
|
|
677
|
+
partNumber: number,
|
|
678
|
+
): string | null => {
|
|
679
|
+
const file = store.filename;
|
|
680
|
+
if (!file || file === ":memory:") return null;
|
|
681
|
+
return join(dirname(resolve(file)), "r2-parts", `${r2PartId(bucketPart, key, uploadId, partNumber)}.part`);
|
|
682
|
+
};
|
|
683
|
+
const r2PartMemKey = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
|
|
684
|
+
`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`;
|
|
685
|
+
const r2WritePart = (
|
|
686
|
+
store: Database,
|
|
687
|
+
bucketPart: string,
|
|
688
|
+
key: string,
|
|
689
|
+
uploadId: string,
|
|
690
|
+
partNumber: number,
|
|
691
|
+
bytes: Uint8Array,
|
|
692
|
+
): void => {
|
|
693
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
694
|
+
if (!path) {
|
|
695
|
+
r2MemParts.set(r2PartMemKey(bucketPart, key, uploadId, partNumber), bytes);
|
|
445
696
|
return;
|
|
446
697
|
}
|
|
447
698
|
mkdirSync(dirname(path), { recursive: true });
|
|
448
|
-
|
|
699
|
+
const temporary = `${path}.tmp`;
|
|
700
|
+
writeFileSync(temporary, bytes);
|
|
701
|
+
renameSync(temporary, path);
|
|
449
702
|
};
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
703
|
+
const r2ReadPart = (
|
|
704
|
+
store: Database,
|
|
705
|
+
bucketPart: string,
|
|
706
|
+
key: string,
|
|
707
|
+
uploadId: string,
|
|
708
|
+
partNumber: number,
|
|
709
|
+
): Uint8Array | null => {
|
|
710
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
711
|
+
if (!path) return r2MemParts.get(r2PartMemKey(bucketPart, key, uploadId, partNumber)) ?? null;
|
|
453
712
|
return existsSync(path) ? readFileSync(path) : null;
|
|
454
713
|
};
|
|
455
|
-
const
|
|
456
|
-
|
|
714
|
+
const r2DeletePart = (
|
|
715
|
+
store: Database,
|
|
716
|
+
bucketPart: string,
|
|
717
|
+
key: string,
|
|
718
|
+
uploadId: string,
|
|
719
|
+
partNumber: number,
|
|
720
|
+
): void => {
|
|
721
|
+
const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
|
|
457
722
|
if (!path) {
|
|
458
|
-
|
|
723
|
+
r2MemParts.delete(r2PartMemKey(bucketPart, key, uploadId, partNumber));
|
|
459
724
|
return;
|
|
460
725
|
}
|
|
461
726
|
if (existsSync(path)) rmSync(path);
|
|
462
727
|
};
|
|
463
728
|
|
|
729
|
+
const requireMultipart = (store: Database, bucket: string, key: string, uploadId: string): R2UploadRow => {
|
|
730
|
+
const row = store
|
|
731
|
+
.query<R2UploadRow, [string, string, string]>(
|
|
732
|
+
"SELECT key, uploaded, http_json, custom_json FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?",
|
|
733
|
+
)
|
|
734
|
+
.get(bucket, key, uploadId);
|
|
735
|
+
if (!row) throw new Error("multipart upload not found");
|
|
736
|
+
return row;
|
|
737
|
+
};
|
|
738
|
+
|
|
739
|
+
const multipartParts = (store: Database, bucket: string, key: string, uploadId: string): R2PartRow[] =>
|
|
740
|
+
store
|
|
741
|
+
.query<R2PartRow, [string, string, string]>(
|
|
742
|
+
"SELECT part_number, size, etag FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ? ORDER BY part_number",
|
|
743
|
+
)
|
|
744
|
+
.all(bucket, key, uploadId);
|
|
745
|
+
|
|
746
|
+
const requestedParts = (value: JsonValue | undefined): Array<{ partNumber: number; etag: string }> => {
|
|
747
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error("complete requires at least one uploaded part");
|
|
748
|
+
return value.map((part) => {
|
|
749
|
+
const parsed = jsonObject(part);
|
|
750
|
+
const partNumber = Number(parsed?.partNumber);
|
|
751
|
+
const etag = isString(parsed?.etag) ? parsed.etag : "";
|
|
752
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000 || !etag) {
|
|
753
|
+
throw new Error("invalid uploaded part");
|
|
754
|
+
}
|
|
755
|
+
return { partNumber, etag };
|
|
756
|
+
});
|
|
757
|
+
};
|
|
758
|
+
|
|
759
|
+
const validateCompletedParts = (stored: R2PartRow[], requested: Array<{ partNumber: number; etag: string }>): void => {
|
|
760
|
+
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
761
|
+
for (let index = 0; index < stored.length; index++) {
|
|
762
|
+
const actual = stored[index];
|
|
763
|
+
const expected = requested[index];
|
|
764
|
+
if (actual.part_number !== expected.partNumber || actual.etag !== expected.etag) {
|
|
765
|
+
throw new Error("uploaded parts do not match");
|
|
766
|
+
}
|
|
767
|
+
if (index > 0 && expected.partNumber <= requested[index - 1].partNumber) {
|
|
768
|
+
throw new Error("uploaded parts must be ordered by partNumber");
|
|
769
|
+
}
|
|
770
|
+
if (index < stored.length - 1 && actual.size !== stored[0].size) {
|
|
771
|
+
throw new Error("all multipart parts except the last must have the same size");
|
|
772
|
+
}
|
|
773
|
+
if (index === stored.length - 1 && stored.length > 1 && actual.size > stored[0].size) {
|
|
774
|
+
throw new Error("the last multipart part cannot be larger than the others");
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
|
|
779
|
+
const deleteMultipart = (store: Database, bucket: string, key: string, uploadId: string): void => {
|
|
780
|
+
for (const part of multipartParts(store, bucket, key, uploadId)) {
|
|
781
|
+
r2DeletePart(store, bucket, key, uploadId, part.part_number);
|
|
782
|
+
}
|
|
783
|
+
store.query("DELETE FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
784
|
+
store.query("DELETE FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
|
|
785
|
+
};
|
|
786
|
+
|
|
787
|
+
const cleanupExpiredMultipart = (store: Database): void => {
|
|
788
|
+
const cutoff = new Date(Date.now() - MULTIPART_EXPIRY_MS).toISOString();
|
|
789
|
+
const expired = store
|
|
790
|
+
.query<R2UploadIdentity, [string]>("SELECT bucket, key, upload_id FROM r2_upload WHERE uploaded < ?")
|
|
791
|
+
.all(cutoff);
|
|
792
|
+
for (const upload of expired) deleteMultipart(store, upload.bucket, upload.key, upload.upload_id);
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
const commitR2Metadata = (
|
|
796
|
+
store: Database,
|
|
797
|
+
bucket: string,
|
|
798
|
+
row: R2Row,
|
|
799
|
+
previous: Pick<R2Row, "blob_id"> | null,
|
|
800
|
+
): void => {
|
|
801
|
+
try {
|
|
802
|
+
store
|
|
803
|
+
.query(
|
|
804
|
+
"INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json, blob_id) " +
|
|
805
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
|
|
806
|
+
"ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, " +
|
|
807
|
+
"custom_json=?7, blob_id=?8",
|
|
808
|
+
)
|
|
809
|
+
.run(bucket, row.key, row.size, row.etag, row.uploaded, row.http_json, row.custom_json, row.blob_id);
|
|
810
|
+
} catch (error) {
|
|
811
|
+
if (previous?.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, row.blob_id);
|
|
812
|
+
throw error;
|
|
813
|
+
}
|
|
814
|
+
if (previous && previous.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, previous.blob_id);
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
const putR2 = (
|
|
818
|
+
store: Database,
|
|
819
|
+
bucket: string,
|
|
820
|
+
key: string,
|
|
821
|
+
bytes: Uint8Array,
|
|
822
|
+
httpJson: string,
|
|
823
|
+
customJson: string,
|
|
824
|
+
): R2Row => {
|
|
825
|
+
const previous = store
|
|
826
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
827
|
+
.get(bucket, key);
|
|
828
|
+
const etag = createHash("sha256").update(bytes).digest("hex");
|
|
829
|
+
const row: R2Row = {
|
|
830
|
+
key,
|
|
831
|
+
size: bytes.byteLength,
|
|
832
|
+
etag,
|
|
833
|
+
uploaded: new Date().toISOString(),
|
|
834
|
+
http_json: httpJson,
|
|
835
|
+
custom_json: customJson,
|
|
836
|
+
blob_id: etag,
|
|
837
|
+
};
|
|
838
|
+
r2WriteBlob(store, bucket, key, bytes, etag);
|
|
839
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
840
|
+
return row;
|
|
841
|
+
};
|
|
842
|
+
|
|
843
|
+
const transferLimit = (): number => {
|
|
844
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
845
|
+
return Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
846
|
+
};
|
|
847
|
+
let activeUploads = 0;
|
|
848
|
+
const maxActiveUploads = 4;
|
|
849
|
+
|
|
850
|
+
const cleanupTransferTemps = (store: Database): void => {
|
|
851
|
+
const directory = r2BlobDir(store);
|
|
852
|
+
if (!directory || !existsSync(directory)) return;
|
|
853
|
+
const now = Date.now();
|
|
854
|
+
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
|
855
|
+
for (const name of readdirSync(directory)) {
|
|
856
|
+
if (!/^[0-9a-f]{24}\.upload\.tmp$/.test(name)) continue;
|
|
857
|
+
const path = join(directory, name);
|
|
858
|
+
try {
|
|
859
|
+
const tokenValue = name.slice(0, 24);
|
|
860
|
+
const ticket = store.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?").get(tokenValue);
|
|
861
|
+
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
862
|
+
} catch (error) {
|
|
863
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
|
|
868
|
+
const transferPath = (binding: string, tokenValue: string): string =>
|
|
869
|
+
`/__sb/r2/transfer/${encodeURIComponent(binding)}/${tokenValue}`;
|
|
870
|
+
|
|
871
|
+
const parseRange = (header: string | null, size: number): { start: number; end: number } | null => {
|
|
872
|
+
if (!header) return { start: 0, end: Math.max(0, size - 1) };
|
|
873
|
+
const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
874
|
+
if (!match || size === 0) return null;
|
|
875
|
+
if (!match[1]) {
|
|
876
|
+
const suffix = Number(match[2]);
|
|
877
|
+
if (!Number.isSafeInteger(suffix) || suffix <= 0) return null;
|
|
878
|
+
return { start: Math.max(0, size - suffix), end: size - 1 };
|
|
879
|
+
}
|
|
880
|
+
const start = Number(match[1]);
|
|
881
|
+
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
882
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return null;
|
|
883
|
+
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
884
|
+
};
|
|
885
|
+
|
|
886
|
+
const transfer = async (request: Request): Promise<Response> => {
|
|
887
|
+
const url = new URL(request.url);
|
|
888
|
+
const match = /^\/__sb\/r2\/transfer\/([A-Z][A-Z0-9_]*)\/([0-9a-f]{24})$/.exec(url.pathname);
|
|
889
|
+
if (!match) return new Response("not found", { status: 404 });
|
|
890
|
+
let store: Database;
|
|
891
|
+
let bucket: string;
|
|
892
|
+
try {
|
|
893
|
+
({ store, part: bucket } = storeFor("r2", requireR2(match[1])));
|
|
894
|
+
} catch {
|
|
895
|
+
return new Response("not found", { status: 404 });
|
|
896
|
+
}
|
|
897
|
+
const tokenValue = match[2];
|
|
898
|
+
const ticket = store
|
|
899
|
+
.query<R2TransferRow, [string]>(
|
|
900
|
+
"SELECT bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json, used " +
|
|
901
|
+
"FROM r2_transfer WHERE token = ?",
|
|
902
|
+
)
|
|
903
|
+
.get(tokenValue);
|
|
904
|
+
const method = request.method.toUpperCase();
|
|
905
|
+
const allowedMethod = ticket?.method === "upload" ? "PUT" : ticket?.method === "download" ? "GET, HEAD" : "";
|
|
906
|
+
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
907
|
+
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
908
|
+
}
|
|
909
|
+
if ((ticket.method === "upload" && method !== "PUT") || (ticket.method === "download" && method !== "GET" && method !== "HEAD")) {
|
|
910
|
+
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
911
|
+
}
|
|
912
|
+
if (ticket.method === "download") {
|
|
913
|
+
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, ticket.key);
|
|
914
|
+
if (!row) return new Response("not found", { status: 404 });
|
|
915
|
+
const etag = `"${row.etag}"`;
|
|
916
|
+
if (request.headers.get("if-none-match") === etag) return new Response(null, { status: 304, headers: { etag } });
|
|
917
|
+
const rangeHeader = request.headers.get("range");
|
|
918
|
+
const range = parseRange(rangeHeader, row.size);
|
|
919
|
+
if (!range) {
|
|
920
|
+
return new Response(null, { status: 416, headers: { "content-range": `bytes */${row.size}`, etag } });
|
|
921
|
+
}
|
|
922
|
+
const path = r2BlobPath(store, bucket, ticket.key, row.blob_id);
|
|
923
|
+
if (!path || !existsSync(path)) return new Response("not found", { status: 404 });
|
|
924
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
925
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
926
|
+
const length = row.size === 0 ? 0 : range.end - range.start + 1;
|
|
927
|
+
const metadata = jsonObject(parseJsonValue(row.http_json));
|
|
928
|
+
const headers = new Headers({
|
|
929
|
+
"accept-ranges": "bytes",
|
|
930
|
+
"content-length": String(length),
|
|
931
|
+
etag,
|
|
932
|
+
});
|
|
933
|
+
if (isString(metadata?.contentType)) headers.set("content-type", metadata.contentType);
|
|
934
|
+
if (rangeHeader) headers.set("content-range", `bytes ${range.start}-${range.end}/${row.size}`);
|
|
935
|
+
const body = method === "HEAD" ? null : Bun.file(path).slice(range.start, range.start + length);
|
|
936
|
+
return new Response(body, { status: rangeHeader ? 206 : 200, headers });
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
const declared = request.headers.has("content-length") ? Number(request.headers.get("content-length")) : NaN;
|
|
940
|
+
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
941
|
+
return new Response("invalid content length", { status: 400 });
|
|
942
|
+
}
|
|
943
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes) return new Response("upload too large", { status: 413 });
|
|
944
|
+
const directory = r2BlobDir(store);
|
|
945
|
+
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
946
|
+
if (activeUploads >= maxActiveUploads) {
|
|
947
|
+
return new Response("too many active uploads", { status: 503, headers: { "retry-after": "5" } });
|
|
948
|
+
}
|
|
949
|
+
const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
|
|
950
|
+
if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
|
|
951
|
+
activeUploads++;
|
|
952
|
+
const temporary = join(directory, `${tokenValue}.upload.tmp`);
|
|
953
|
+
await (async () => {
|
|
954
|
+
mkdirSync(directory, { recursive: true });
|
|
955
|
+
const reservation = await open(temporary, "wx");
|
|
956
|
+
await reservation.close();
|
|
957
|
+
})().catch((error) => {
|
|
958
|
+
activeUploads--;
|
|
959
|
+
throw error;
|
|
960
|
+
});
|
|
961
|
+
const writer = Bun.file(temporary).writer({ highWaterMark: 64 * 1024 });
|
|
962
|
+
const hash = createHash("sha256");
|
|
963
|
+
let size = 0;
|
|
964
|
+
try {
|
|
965
|
+
if (request.body) {
|
|
966
|
+
for await (const chunk of request.body) {
|
|
967
|
+
size += chunk.byteLength;
|
|
968
|
+
if (size > ticket.max_bytes) throw new RangeError("upload too large");
|
|
969
|
+
const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
|
|
970
|
+
if (floor > 0 && availableBytes() < chunk.byteLength + floor) {
|
|
971
|
+
const error = new Error("R2 storage is temporarily unavailable");
|
|
972
|
+
Object.assign(error, { code: "SB_R2_DISK_FLOOR" });
|
|
973
|
+
throw error;
|
|
974
|
+
}
|
|
975
|
+
hash.update(chunk);
|
|
976
|
+
const written = await writer.write(chunk);
|
|
977
|
+
if (written !== chunk.byteLength) throw new Error("upload write was incomplete");
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
await writer.end();
|
|
981
|
+
const persisted = await open(temporary, "r");
|
|
982
|
+
try {
|
|
983
|
+
await persisted.sync();
|
|
984
|
+
} finally {
|
|
985
|
+
await persisted.close();
|
|
986
|
+
}
|
|
987
|
+
const etag = hash.digest("hex");
|
|
988
|
+
if (ticket.expected_sha256 && ticket.expected_sha256 !== etag) {
|
|
989
|
+
rmSync(temporary);
|
|
990
|
+
settleR2(tokenValue);
|
|
991
|
+
return Response.json({ error: "sha256 mismatch", actual: etag }, { status: 422 });
|
|
992
|
+
}
|
|
993
|
+
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
994
|
+
renameSync(temporary, destination);
|
|
995
|
+
const previous = store
|
|
996
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>("SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?")
|
|
997
|
+
.get(bucket, ticket.key);
|
|
998
|
+
const row: R2Row = {
|
|
999
|
+
key: ticket.key,
|
|
1000
|
+
size,
|
|
1001
|
+
etag,
|
|
1002
|
+
uploaded: new Date().toISOString(),
|
|
1003
|
+
http_json: ticket.http_json,
|
|
1004
|
+
custom_json: ticket.custom_json,
|
|
1005
|
+
blob_id: etag,
|
|
1006
|
+
};
|
|
1007
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
1008
|
+
settleR2(tokenValue);
|
|
1009
|
+
return Response.json({ object: r2Row(row) }, { status: 201, headers: { etag: `"${etag}"` } });
|
|
1010
|
+
} catch (error) {
|
|
1011
|
+
try {
|
|
1012
|
+
await writer.end();
|
|
1013
|
+
} catch {
|
|
1014
|
+
// Preserve the original upload failure while releasing the file sink.
|
|
1015
|
+
}
|
|
1016
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
1017
|
+
settleR2(tokenValue);
|
|
1018
|
+
if (error instanceof RangeError) {
|
|
1019
|
+
recordR2TransferMetric("rejected_size");
|
|
1020
|
+
return new Response(error.message, { status: 413 });
|
|
1021
|
+
}
|
|
1022
|
+
if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "SB_R2_DISK_FLOOR")) {
|
|
1023
|
+
recordR2TransferMetric("rejected_disk");
|
|
1024
|
+
return new Response("R2 storage is temporarily unavailable", { status: 507 });
|
|
1025
|
+
}
|
|
1026
|
+
throw error;
|
|
1027
|
+
} finally {
|
|
1028
|
+
activeUploads--;
|
|
1029
|
+
}
|
|
1030
|
+
};
|
|
1031
|
+
|
|
1032
|
+
const completeMultipart = (
|
|
1033
|
+
store: Database,
|
|
1034
|
+
bucket: string,
|
|
1035
|
+
key: string,
|
|
1036
|
+
uploadId: string,
|
|
1037
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
1038
|
+
): ReturnType<typeof r2Row> => {
|
|
1039
|
+
const upload = requireMultipart(store, bucket, key, uploadId);
|
|
1040
|
+
const parts = multipartParts(store, bucket, key, uploadId);
|
|
1041
|
+
validateCompletedParts(parts, requested);
|
|
1042
|
+
|
|
1043
|
+
const hash = createHash("sha256");
|
|
1044
|
+
let size = 0;
|
|
1045
|
+
const blobDirectory = r2BlobDir(store);
|
|
1046
|
+
let inMemoryBody: Uint8Array | null = null;
|
|
1047
|
+
let temporary: string | null = null;
|
|
1048
|
+
if (blobDirectory) {
|
|
1049
|
+
mkdirSync(blobDirectory, { recursive: true });
|
|
1050
|
+
temporary = join(blobDirectory, `${uploadId}.tmp`);
|
|
1051
|
+
writeFileSync(temporary, new Uint8Array(0));
|
|
1052
|
+
try {
|
|
1053
|
+
for (const part of parts) {
|
|
1054
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1055
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1056
|
+
appendFileSync(temporary, bytes);
|
|
1057
|
+
hash.update(bytes);
|
|
1058
|
+
size += bytes.byteLength;
|
|
1059
|
+
}
|
|
1060
|
+
} catch (error) {
|
|
1061
|
+
if (existsSync(temporary)) rmSync(temporary);
|
|
1062
|
+
throw error;
|
|
1063
|
+
}
|
|
1064
|
+
} else {
|
|
1065
|
+
const chunks: Uint8Array[] = [];
|
|
1066
|
+
for (const part of parts) {
|
|
1067
|
+
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1068
|
+
if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1069
|
+
chunks.push(bytes);
|
|
1070
|
+
hash.update(bytes);
|
|
1071
|
+
size += bytes.byteLength;
|
|
1072
|
+
}
|
|
1073
|
+
inMemoryBody = Buffer.concat(chunks);
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
const etag = hash.digest("hex");
|
|
1077
|
+
const previous = store
|
|
1078
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1079
|
+
.get(bucket, key);
|
|
1080
|
+
if (temporary) renameSync(temporary, r2BlobPath(store, bucket, key, etag)!);
|
|
1081
|
+
else r2WriteBlob(store, bucket, key, inMemoryBody ?? new Uint8Array(0), etag);
|
|
1082
|
+
const row: R2Row = {
|
|
1083
|
+
key,
|
|
1084
|
+
size,
|
|
1085
|
+
etag,
|
|
1086
|
+
uploaded: new Date().toISOString(),
|
|
1087
|
+
http_json: upload.http_json,
|
|
1088
|
+
custom_json: upload.custom_json,
|
|
1089
|
+
blob_id: etag,
|
|
1090
|
+
};
|
|
1091
|
+
commitR2Metadata(store, bucket, row, previous);
|
|
1092
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1093
|
+
return r2Row(row);
|
|
1094
|
+
};
|
|
1095
|
+
|
|
464
1096
|
const r2Row = (r: R2Row) => ({
|
|
465
1097
|
key: r.key,
|
|
466
1098
|
size: r.size,
|
|
@@ -580,12 +1212,18 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
580
1212
|
return { ok: true, op: "pong", echo: msg.msg ?? null, pid: process.pid };
|
|
581
1213
|
case "kv.get": {
|
|
582
1214
|
const { store, part } = storeFor("kv", requireKv(msg.ns));
|
|
583
|
-
const
|
|
584
|
-
|
|
1215
|
+
const stmts = kvStmts(store);
|
|
1216
|
+
const row = stmts.get.get(part, str(msg.key));
|
|
1217
|
+
if (!row) return { ok: true, found: false, value: null };
|
|
1218
|
+
if (row.expires_at != null && row.expires_at <= Date.now()) {
|
|
1219
|
+
stmts.del.run(part, str(msg.key));
|
|
1220
|
+
return { ok: true, found: false, value: null };
|
|
1221
|
+
}
|
|
1222
|
+
return { ok: true, found: true, value: row.value };
|
|
585
1223
|
}
|
|
586
1224
|
case "kv.put": {
|
|
587
1225
|
const { store, part } = storeFor("kv", requireKv(msg.ns));
|
|
588
|
-
kvStmts(store).put.run(part, str(msg.key), str(msg.value));
|
|
1226
|
+
kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg));
|
|
589
1227
|
return { ok: true };
|
|
590
1228
|
}
|
|
591
1229
|
case "kv.delete": {
|
|
@@ -598,7 +1236,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
598
1236
|
return {
|
|
599
1237
|
ok: true,
|
|
600
1238
|
keys: kvStmts(store)
|
|
601
|
-
.list.all(part, str(msg.prefix))
|
|
1239
|
+
.list.all(part, str(msg.prefix), Date.now())
|
|
602
1240
|
.map((r) => r.key),
|
|
603
1241
|
};
|
|
604
1242
|
}
|
|
@@ -656,24 +1294,107 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
656
1294
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
657
1295
|
const key = str(msg.key);
|
|
658
1296
|
const bytes = new TextEncoder().encode(str(msg.body));
|
|
659
|
-
|
|
1297
|
+
return {
|
|
1298
|
+
ok: true,
|
|
1299
|
+
object: r2Row(
|
|
1300
|
+
putR2(
|
|
1301
|
+
store,
|
|
1302
|
+
bucket,
|
|
1303
|
+
key,
|
|
1304
|
+
bytes,
|
|
1305
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1306
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1307
|
+
),
|
|
1308
|
+
),
|
|
1309
|
+
};
|
|
1310
|
+
}
|
|
1311
|
+
case "r2.multipart.create": {
|
|
1312
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1313
|
+
cleanupExpiredMultipart(store);
|
|
1314
|
+
const key = str(msg.key);
|
|
1315
|
+
const uploadId = str(msg.uploadId) || newId();
|
|
660
1316
|
const uploaded = new Date().toISOString();
|
|
661
|
-
r2WriteBlob(store, bucket, key, bytes);
|
|
662
1317
|
store
|
|
663
1318
|
.query(
|
|
664
|
-
"INSERT INTO
|
|
665
|
-
"ON CONFLICT (bucket, key) DO
|
|
1319
|
+
"INSERT INTO r2_upload (bucket, key, upload_id, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1320
|
+
"ON CONFLICT (bucket, key, upload_id) DO NOTHING",
|
|
666
1321
|
)
|
|
667
1322
|
.run(
|
|
668
1323
|
bucket,
|
|
669
1324
|
key,
|
|
670
|
-
|
|
671
|
-
etag,
|
|
1325
|
+
uploadId,
|
|
672
1326
|
uploaded,
|
|
673
1327
|
JSON.stringify(msg.httpMetadata ?? {}),
|
|
674
1328
|
JSON.stringify(msg.customMetadata ?? {}),
|
|
675
1329
|
);
|
|
676
|
-
return { ok: true,
|
|
1330
|
+
return { ok: true, key, uploadId };
|
|
1331
|
+
}
|
|
1332
|
+
case "r2.transfer.create": {
|
|
1333
|
+
const binding = requireR2(msg.bucket);
|
|
1334
|
+
const { store, part: bucket } = storeFor("r2", binding);
|
|
1335
|
+
const key = str(msg.key);
|
|
1336
|
+
const method = msg.method === "download" ? "download" : msg.method === "upload" ? "upload" : "";
|
|
1337
|
+
if (!method) throw new Error("transfer method must be upload or download");
|
|
1338
|
+
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1339
|
+
const expires = Date.now() + expiresIn * 1000;
|
|
1340
|
+
const maxBytes =
|
|
1341
|
+
method === "upload"
|
|
1342
|
+
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1343
|
+
: 0;
|
|
1344
|
+
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1345
|
+
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1346
|
+
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
1347
|
+
}
|
|
1348
|
+
store.query("DELETE FROM r2_transfer WHERE expires < ? OR used = 1").run(Date.now());
|
|
1349
|
+
cleanupTransferTemps(store);
|
|
1350
|
+
const tokenValue = randomBytes(12).toString("hex");
|
|
1351
|
+
const reservation = method === "upload" ? reserveR2(tokenValue, maxBytes, expires) : "ok";
|
|
1352
|
+
if (reservation === "quota") {
|
|
1353
|
+
recordR2TransferMetric("rejected_quota");
|
|
1354
|
+
throw new Error("R2 account quota would be exceeded");
|
|
1355
|
+
}
|
|
1356
|
+
if (reservation === "disk") {
|
|
1357
|
+
recordR2TransferMetric("rejected_disk");
|
|
1358
|
+
throw new Error("R2 storage is temporarily unavailable");
|
|
1359
|
+
}
|
|
1360
|
+
try {
|
|
1361
|
+
store
|
|
1362
|
+
.query(
|
|
1363
|
+
"INSERT INTO r2_transfer (token, bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json) " +
|
|
1364
|
+
"VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
|
|
1365
|
+
)
|
|
1366
|
+
.run(
|
|
1367
|
+
tokenValue,
|
|
1368
|
+
bucket,
|
|
1369
|
+
key,
|
|
1370
|
+
method,
|
|
1371
|
+
expires,
|
|
1372
|
+
maxBytes,
|
|
1373
|
+
expectedSha256,
|
|
1374
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1375
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1376
|
+
);
|
|
1377
|
+
} catch (error) {
|
|
1378
|
+
if (reservation === "ok" && method === "upload") settleR2(tokenValue);
|
|
1379
|
+
throw error;
|
|
1380
|
+
}
|
|
1381
|
+
recordR2TransferMetric("tickets_issued");
|
|
1382
|
+
return { ok: true, url: transferPath(binding, tokenValue), expiresAt: new Date(expires).toISOString() };
|
|
1383
|
+
}
|
|
1384
|
+
case "r2.multipart.complete": {
|
|
1385
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1386
|
+
const key = str(msg.key);
|
|
1387
|
+
const uploadId = str(msg.uploadId);
|
|
1388
|
+
const object = completeMultipart(store, bucket, key, uploadId, requestedParts(msg.parts));
|
|
1389
|
+
return { ok: true, object };
|
|
1390
|
+
}
|
|
1391
|
+
case "r2.multipart.abort": {
|
|
1392
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1393
|
+
const key = str(msg.key);
|
|
1394
|
+
const uploadId = str(msg.uploadId);
|
|
1395
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
1396
|
+
deleteMultipart(store, bucket, key, uploadId);
|
|
1397
|
+
return { ok: true };
|
|
677
1398
|
}
|
|
678
1399
|
case "r2.get":
|
|
679
1400
|
case "r2.head": {
|
|
@@ -681,15 +1402,19 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
681
1402
|
const key = str(msg.key);
|
|
682
1403
|
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
683
1404
|
if (!row) return { ok: true, found: false };
|
|
684
|
-
const body =
|
|
685
|
-
|
|
1405
|
+
const body = msg.op === "r2.get"
|
|
1406
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1407
|
+
: undefined;
|
|
686
1408
|
return { ok: true, found: true, object: r2Row(row), body };
|
|
687
1409
|
}
|
|
688
1410
|
case "r2.delete": {
|
|
689
1411
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
690
1412
|
const key = str(msg.key);
|
|
1413
|
+
const row = store
|
|
1414
|
+
.query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
|
|
1415
|
+
.get(bucket, key);
|
|
691
1416
|
store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
|
|
692
|
-
r2DeleteBlob(store, bucket, key);
|
|
1417
|
+
if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
|
|
693
1418
|
return { ok: true };
|
|
694
1419
|
}
|
|
695
1420
|
case "r2.list": {
|
|
@@ -876,24 +1601,35 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
876
1601
|
// second full-object copy through the database's own page cache.
|
|
877
1602
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
878
1603
|
const key = str(msg.key);
|
|
1604
|
+
const row = putR2(
|
|
1605
|
+
store,
|
|
1606
|
+
bucket,
|
|
1607
|
+
key,
|
|
1608
|
+
binary,
|
|
1609
|
+
JSON.stringify(msg.httpMetadata ?? {}),
|
|
1610
|
+
JSON.stringify(msg.customMetadata ?? {}),
|
|
1611
|
+
);
|
|
1612
|
+
return { reply: { ok: true, object: r2Row(row) } };
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
if (msg.op === "r2.multipart.put") {
|
|
1616
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1617
|
+
const key = str(msg.key);
|
|
1618
|
+
const uploadId = str(msg.uploadId);
|
|
1619
|
+
const partNumber = Number(msg.partNumber);
|
|
1620
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
|
|
1621
|
+
throw new Error("partNumber must be an integer between 1 and 10000");
|
|
1622
|
+
}
|
|
1623
|
+
requireMultipart(store, bucket, key, uploadId);
|
|
879
1624
|
const etag = createHash("sha256").update(binary).digest("hex");
|
|
880
|
-
|
|
881
|
-
r2WriteBlob(store, bucket, key, binary);
|
|
1625
|
+
r2WritePart(store, bucket, key, uploadId, partNumber, binary);
|
|
882
1626
|
store
|
|
883
1627
|
.query(
|
|
884
|
-
"INSERT INTO
|
|
885
|
-
"ON CONFLICT (bucket, key) DO UPDATE SET size=?
|
|
1628
|
+
"INSERT INTO r2_part (bucket, key, upload_id, part_number, size, etag) VALUES (?1,?2,?3,?4,?5,?6) " +
|
|
1629
|
+
"ON CONFLICT (bucket, key, upload_id, part_number) DO UPDATE SET size=?5, etag=?6",
|
|
886
1630
|
)
|
|
887
|
-
.run(
|
|
888
|
-
|
|
889
|
-
key,
|
|
890
|
-
binary.byteLength,
|
|
891
|
-
etag,
|
|
892
|
-
uploaded,
|
|
893
|
-
JSON.stringify(msg.httpMetadata ?? {}),
|
|
894
|
-
JSON.stringify(msg.customMetadata ?? {}),
|
|
895
|
-
);
|
|
896
|
-
return { reply: { ok: true, object: { key, size: binary.byteLength, etag, uploaded } } };
|
|
1631
|
+
.run(bucket, key, uploadId, partNumber, binary.byteLength, etag);
|
|
1632
|
+
return { reply: { ok: true, part: { partNumber, etag } } };
|
|
897
1633
|
}
|
|
898
1634
|
|
|
899
1635
|
if (msg.op === "r2.get") {
|
|
@@ -901,7 +1637,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
901
1637
|
const key = str(msg.key);
|
|
902
1638
|
const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
|
|
903
1639
|
if (!row) return { reply: { ok: true, found: false } };
|
|
904
|
-
return {
|
|
1640
|
+
return {
|
|
1641
|
+
reply: { ok: true, found: true, object: r2Row(row) },
|
|
1642
|
+
bytes: r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0),
|
|
1643
|
+
};
|
|
905
1644
|
}
|
|
906
1645
|
|
|
907
1646
|
if (msg.op === "assets.get") {
|
|
@@ -939,7 +1678,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
939
1678
|
if (!msg) throw new Error("request frame was not a JSON object");
|
|
940
1679
|
if (token && str(msg.token) !== token) return frameOf(encodeV1({ ok: false, error: "unauthorized" }));
|
|
941
1680
|
const binary = new Uint8Array(payload.subarray(5 + jsonLen));
|
|
1681
|
+
const id = isSafeInteger(msg.id) ? msg.id : null;
|
|
1682
|
+
const replayKey = id === null ? null : `${str(msg.op)}:${id}`;
|
|
1683
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) {
|
|
1684
|
+
const seen = replayed.get(replayKey);
|
|
1685
|
+
if (seen) return frameOf(encodeV1(seen));
|
|
1686
|
+
}
|
|
942
1687
|
const { reply, bytes } = await dispatchBinary(msg, binary);
|
|
1688
|
+
if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
|
|
943
1689
|
return frameOf(encodeV1(reply, bytes));
|
|
944
1690
|
} catch (e) {
|
|
945
1691
|
return frameOf(encodeV1({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
@@ -966,6 +1712,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
966
1712
|
"d1.backup",
|
|
967
1713
|
"ratelimit.check",
|
|
968
1714
|
"r2.put",
|
|
1715
|
+
"r2.multipart.create",
|
|
1716
|
+
"r2.multipart.put",
|
|
1717
|
+
"r2.multipart.complete",
|
|
1718
|
+
"r2.multipart.abort",
|
|
1719
|
+
"r2.transfer.create",
|
|
969
1720
|
"r2.delete",
|
|
970
1721
|
"do.storage.put",
|
|
971
1722
|
"do.storage.delete",
|
|
@@ -979,6 +1730,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
979
1730
|
* milliseconds, so this only has to outlive a reconnect, not a session. */
|
|
980
1731
|
const REPLAY_WINDOW = 256;
|
|
981
1732
|
|
|
1733
|
+
const rememberReplay = (key: string, reply: Frame): void => {
|
|
1734
|
+
replayed.set(key, reply);
|
|
1735
|
+
if (replayed.size > REPLAY_WINDOW) {
|
|
1736
|
+
const oldest = replayed.keys().next().value;
|
|
1737
|
+
if (oldest !== undefined) replayed.delete(oldest);
|
|
1738
|
+
}
|
|
1739
|
+
};
|
|
1740
|
+
|
|
982
1741
|
/**
|
|
983
1742
|
* Run `msg`, or replay what it answered last time. Keyed by the id the sprout
|
|
984
1743
|
* put on the request; a request without one is always run.
|
|
@@ -991,12 +1750,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
991
1750
|
const seen = replayed.get(key);
|
|
992
1751
|
if (seen) return seen;
|
|
993
1752
|
const reply = await dispatch(msg);
|
|
994
|
-
|
|
995
|
-
// Insertion-ordered, so the oldest key is the first one out.
|
|
996
|
-
if (replayed.size > REPLAY_WINDOW) {
|
|
997
|
-
const oldest = replayed.keys().next().value;
|
|
998
|
-
if (oldest !== undefined) replayed.delete(oldest);
|
|
999
|
-
}
|
|
1753
|
+
rememberReplay(key, reply);
|
|
1000
1754
|
return reply;
|
|
1001
1755
|
}
|
|
1002
1756
|
|
|
@@ -1143,6 +1897,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1143
1897
|
dispatch,
|
|
1144
1898
|
handlePayload,
|
|
1145
1899
|
handleFrame,
|
|
1900
|
+
transfer,
|
|
1146
1901
|
close: () => {
|
|
1147
1902
|
for (const t of timers) clearInterval(t);
|
|
1148
1903
|
for (const conn of d1Conns.values()) conn.close();
|
|
@@ -1230,17 +1985,33 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
1230
1985
|
},
|
|
1231
1986
|
},
|
|
1232
1987
|
});
|
|
1233
|
-
return { port: server.port, stop: () => server.stop(true) };
|
|
1988
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
/** Start the HTTP listener that serves short-lived direct R2 transfer tickets. */
|
|
1992
|
+
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
1993
|
+
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
1994
|
+
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
1995
|
+
const server = Bun.serve({ hostname, port, maxRequestBodySize, fetch(request, server) {
|
|
1996
|
+
server.timeout(request, 255);
|
|
1997
|
+
return broker.transfer(request);
|
|
1998
|
+
} });
|
|
1999
|
+
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
1234
2000
|
}
|
|
1235
2001
|
|
|
1236
2002
|
if (import.meta.main) {
|
|
1237
2003
|
const { values } = parseArgs({
|
|
1238
2004
|
options: {
|
|
1239
2005
|
port: { type: "string" },
|
|
2006
|
+
"transfer-port": { type: "string" },
|
|
1240
2007
|
token: { type: "string" },
|
|
1241
2008
|
db: { type: "string" },
|
|
1242
2009
|
"data-dir": { type: "string" },
|
|
1243
2010
|
"resource-dir": { type: "string" },
|
|
2011
|
+
"owner-id": { type: "string" },
|
|
2012
|
+
"r2-quota-bytes": { type: "string" },
|
|
2013
|
+
"r2-resource-ids": { type: "string" },
|
|
2014
|
+
"r2-min-free-bytes": { type: "string" },
|
|
1244
2015
|
bindings: { type: "string" },
|
|
1245
2016
|
secrets: { type: "string" },
|
|
1246
2017
|
"sprout-url": { type: "string" },
|
|
@@ -1289,6 +2060,10 @@ if (import.meta.main) {
|
|
|
1289
2060
|
db: values.db,
|
|
1290
2061
|
dataDir: values["data-dir"],
|
|
1291
2062
|
resourceDir: values["resource-dir"],
|
|
2063
|
+
ownerId: values["owner-id"],
|
|
2064
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2065
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2066
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
1292
2067
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
1293
2068
|
bindings,
|
|
1294
2069
|
secrets,
|
|
@@ -1299,7 +2074,9 @@ if (import.meta.main) {
|
|
|
1299
2074
|
dispatchEnabled: () => dispatchEnabled,
|
|
1300
2075
|
});
|
|
1301
2076
|
const { port } = listen(broker, "127.0.0.1", Number(values.port ?? process.env.SB_BROKER_PORT ?? 0));
|
|
2077
|
+
const transferPort = Number(values["transfer-port"] ?? process.env.SB_R2_TRANSFER_PORT ?? 0);
|
|
2078
|
+
const transferServer = transferPort > 0 ? listenTransfers(broker, "127.0.0.1", transferPort) : null;
|
|
1302
2079
|
console.log(
|
|
1303
|
-
`sproutboat broker: 127.0.0.1:${port} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
2080
|
+
`sproutboat broker: 127.0.0.1:${port} transfer=${transferServer?.port ?? "off"} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
|
|
1304
2081
|
);
|
|
1305
2082
|
}
|