@sproutboat/wire 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/package.json +2 -2
- package/src/broker.test.ts +327 -97
- package/src/broker.ts +323 -170
package/src/broker.ts
CHANGED
|
@@ -21,7 +21,18 @@ import { createD1Store } from "./d1";
|
|
|
21
21
|
import type { D1Pool } from "./d1-pool";
|
|
22
22
|
import { Database, type Statement } from "bun:sqlite";
|
|
23
23
|
import { createHash, randomBytes } from "node:crypto";
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
appendFileSync,
|
|
26
|
+
existsSync,
|
|
27
|
+
mkdirSync,
|
|
28
|
+
readFileSync,
|
|
29
|
+
readdirSync,
|
|
30
|
+
renameSync,
|
|
31
|
+
rmSync,
|
|
32
|
+
statfsSync,
|
|
33
|
+
statSync,
|
|
34
|
+
writeFileSync,
|
|
35
|
+
} from "node:fs";
|
|
25
36
|
import { open } from "node:fs/promises";
|
|
26
37
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
27
38
|
import { parseArgs } from "node:util";
|
|
@@ -200,7 +211,12 @@ export type Broker = {
|
|
|
200
211
|
* for it (`bodyStream: true`) sends that stream as its request body: the listener feeds
|
|
201
212
|
* it from the FRAME_V1_MORE frames that follow the head on the same connection.
|
|
202
213
|
*/
|
|
203
|
-
handleFrame(
|
|
214
|
+
handleFrame(
|
|
215
|
+
payload: Buffer,
|
|
216
|
+
signal?: AbortSignal,
|
|
217
|
+
emit?: FrameEmitter,
|
|
218
|
+
body?: ReadableStream<Uint8Array>,
|
|
219
|
+
): Promise<Buffer>;
|
|
204
220
|
/** Serve one reserved direct R2 upload or download request. */
|
|
205
221
|
transfer(request: Request): Promise<Response>;
|
|
206
222
|
close(): void;
|
|
@@ -300,10 +316,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
300
316
|
"expires_at INTEGER, PRIMARY KEY (ns, key))",
|
|
301
317
|
);
|
|
302
318
|
const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
|
|
303
|
-
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
319
|
+
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
320
|
+
conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
|
|
304
321
|
// #233 — 1 when the value was put as bytes (stored one char per byte), so get()
|
|
305
322
|
// can hand back exact bytes or decoded text depending on the requested type.
|
|
306
|
-
if (!kvColumns.some((column) => column.name === "binary"))
|
|
323
|
+
if (!kvColumns.some((column) => column.name === "binary"))
|
|
324
|
+
conn.exec("ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0");
|
|
307
325
|
// A key's metadata (put(key, value, { metadata })), as JSON text.
|
|
308
326
|
if (!kvColumns.some((column) => column.name === "metadata")) conn.exec("ALTER TABLE kv ADD COLUMN metadata TEXT");
|
|
309
327
|
// #56 — no `body` column: an object's bytes live in their own file under
|
|
@@ -344,7 +362,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
344
362
|
// Every queue poll filters one partition's live rows and takes its earliest
|
|
345
363
|
// visible batch. This additive index also upgrades existing resource files.
|
|
346
364
|
conn.exec("CREATE INDEX IF NOT EXISTS mq_due ON mq (queue, dead, visible_at)");
|
|
347
|
-
if (
|
|
365
|
+
if (
|
|
366
|
+
!conn
|
|
367
|
+
.query<{ name: string }, []>("PRAGMA table_info(mq)")
|
|
368
|
+
.all()
|
|
369
|
+
.some((c) => c.name === "claim_token")
|
|
370
|
+
)
|
|
348
371
|
conn.exec("ALTER TABLE mq ADD COLUMN claim_token TEXT NOT NULL DEFAULT ''");
|
|
349
372
|
return conn;
|
|
350
373
|
};
|
|
@@ -373,7 +396,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
373
396
|
);
|
|
374
397
|
// Alarm polling reads the earliest due objects across classes.
|
|
375
398
|
doDb.exec("CREATE INDEX IF NOT EXISTS do_alarm_due ON do_alarm (at)");
|
|
376
|
-
doDb.exec(
|
|
399
|
+
doDb.exec(
|
|
400
|
+
"CREATE TABLE IF NOT EXISTS do_alarm_delivery (cls TEXT NOT NULL, id TEXT NOT NULL, at INTEGER NOT NULL, attempts INTEGER NOT NULL, token TEXT NOT NULL, expires INTEGER NOT NULL, PRIMARY KEY(cls,id))",
|
|
401
|
+
);
|
|
377
402
|
// #69 — fixed-window rate-limiter counters, per binding name + key.
|
|
378
403
|
db.exec(
|
|
379
404
|
"CREATE TABLE IF NOT EXISTS ratelimit (name TEXT NOT NULL, key TEXT NOT NULL, window_start INTEGER NOT NULL, " +
|
|
@@ -407,9 +432,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
407
432
|
// One ledger for every account, shared by the brokers of all its deployments.
|
|
408
433
|
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
409
434
|
// requests cannot each independently decide that the same remaining bytes fit.
|
|
410
|
-
const quotaDb =
|
|
411
|
-
|
|
412
|
-
|
|
435
|
+
const quotaDb =
|
|
436
|
+
quotaOwner && quotaDirectory && quotaBytes
|
|
437
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
438
|
+
: null;
|
|
413
439
|
if (quotaDb) {
|
|
414
440
|
const owner = quotaOwner!;
|
|
415
441
|
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
@@ -422,7 +448,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
422
448
|
quotaDb.exec(
|
|
423
449
|
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
424
450
|
);
|
|
425
|
-
const existing = quotaDb
|
|
451
|
+
const existing = quotaDb
|
|
452
|
+
.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?")
|
|
453
|
+
.get(owner);
|
|
426
454
|
if (!existing) {
|
|
427
455
|
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
428
456
|
}
|
|
@@ -436,9 +464,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
436
464
|
// Open through resourceDb so older resource files gain the multipart
|
|
437
465
|
// tables before the aggregate is queried.
|
|
438
466
|
const store = resourceDb(id);
|
|
439
|
-
used +=
|
|
440
|
-
|
|
441
|
-
|
|
467
|
+
used +=
|
|
468
|
+
store
|
|
469
|
+
.query<{ bytes: number }, []>(
|
|
470
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
471
|
+
)
|
|
472
|
+
.get()?.bytes ?? 0;
|
|
442
473
|
}
|
|
443
474
|
return used;
|
|
444
475
|
};
|
|
@@ -453,9 +484,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
453
484
|
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
454
485
|
if (!quotaDb || !quotaOwner) return;
|
|
455
486
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
456
|
-
const reserved =
|
|
457
|
-
|
|
458
|
-
|
|
487
|
+
const reserved =
|
|
488
|
+
quotaDb
|
|
489
|
+
.query<{ bytes: number }, [string]>(
|
|
490
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
491
|
+
)
|
|
492
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
459
493
|
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
460
494
|
};
|
|
461
495
|
|
|
@@ -465,10 +499,15 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
465
499
|
try {
|
|
466
500
|
releaseExpiredR2Reservations();
|
|
467
501
|
const used = committedR2Bytes();
|
|
468
|
-
const reserved =
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
502
|
+
const reserved =
|
|
503
|
+
quotaDb
|
|
504
|
+
.query<{ bytes: number }, [string]>(
|
|
505
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
506
|
+
)
|
|
507
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
508
|
+
quotaDb
|
|
509
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
510
|
+
.run(used, reserved, quotaOwner);
|
|
472
511
|
if (used + reserved + bytes > quotaBytes) {
|
|
473
512
|
quotaDb.exec("ROLLBACK");
|
|
474
513
|
return "quota";
|
|
@@ -478,8 +517,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
478
517
|
quotaDb.exec("ROLLBACK");
|
|
479
518
|
return "disk";
|
|
480
519
|
}
|
|
481
|
-
quotaDb
|
|
482
|
-
|
|
520
|
+
quotaDb
|
|
521
|
+
.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)")
|
|
522
|
+
.run(token, quotaOwner, bytes, expires);
|
|
523
|
+
quotaDb
|
|
524
|
+
.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2")
|
|
525
|
+
.run(reserved + bytes, quotaOwner);
|
|
483
526
|
quotaDb.exec("COMMIT");
|
|
484
527
|
return "ok";
|
|
485
528
|
} catch (error) {
|
|
@@ -492,17 +535,24 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
492
535
|
if (!quotaDb || !quotaOwner) return;
|
|
493
536
|
quotaDb.exec("BEGIN IMMEDIATE");
|
|
494
537
|
try {
|
|
495
|
-
const reservation = quotaDb
|
|
538
|
+
const reservation = quotaDb
|
|
539
|
+
.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?")
|
|
540
|
+
.get(token);
|
|
496
541
|
if (!reservation) {
|
|
497
542
|
quotaDb.exec("COMMIT");
|
|
498
543
|
return;
|
|
499
544
|
}
|
|
500
545
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
501
546
|
const used = committedR2Bytes();
|
|
502
|
-
const reserved =
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
547
|
+
const reserved =
|
|
548
|
+
quotaDb
|
|
549
|
+
.query<{ bytes: number }, [string]>(
|
|
550
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
551
|
+
)
|
|
552
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
553
|
+
quotaDb
|
|
554
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
555
|
+
.run(used, reserved, quotaOwner);
|
|
506
556
|
quotaDb.exec("COMMIT");
|
|
507
557
|
} catch (error) {
|
|
508
558
|
quotaDb.exec("ROLLBACK");
|
|
@@ -510,7 +560,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
510
560
|
}
|
|
511
561
|
};
|
|
512
562
|
|
|
513
|
-
const recordR2TransferMetric = (
|
|
563
|
+
const recordR2TransferMetric = (
|
|
564
|
+
field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size",
|
|
565
|
+
): void => {
|
|
514
566
|
if (!quotaDb || !quotaOwner) return;
|
|
515
567
|
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
516
568
|
};
|
|
@@ -528,19 +580,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
528
580
|
};
|
|
529
581
|
|
|
530
582
|
type KvStmts = {
|
|
531
|
-
get: Statement<
|
|
583
|
+
get: Statement<
|
|
584
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
585
|
+
[string, string]
|
|
586
|
+
>;
|
|
532
587
|
put: Statement<unknown, [string, string, string, number | null, number, string | null]>;
|
|
533
588
|
del: Statement<unknown, [string, string]>;
|
|
534
|
-
list: Statement<
|
|
589
|
+
list: Statement<
|
|
590
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
591
|
+
[string, string, string, number, number]
|
|
592
|
+
>;
|
|
535
593
|
};
|
|
536
594
|
const kvStmtCache = new Map<Database, KvStmts>();
|
|
537
595
|
const kvStmts = (store: Database): KvStmts => {
|
|
538
596
|
let stmts = kvStmtCache.get(store);
|
|
539
597
|
if (!stmts) {
|
|
540
598
|
stmts = {
|
|
541
|
-
get: store.query<
|
|
542
|
-
|
|
543
|
-
|
|
599
|
+
get: store.query<
|
|
600
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
601
|
+
[string, string]
|
|
602
|
+
>("SELECT value, expires_at, binary, metadata FROM kv WHERE ns = ? AND key = ?"),
|
|
544
603
|
put: store.query(
|
|
545
604
|
"INSERT INTO kv (ns, key, value, expires_at, binary, metadata) VALUES (?1, ?2, ?3, ?4, ?5, ?6) " +
|
|
546
605
|
"ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5, metadata = ?6",
|
|
@@ -550,7 +609,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
550
609
|
// it is a separate concern (ponytail: rows accumulate until touched by a
|
|
551
610
|
// get/list/put on the same key/prefix, add a cron sweep if that matters).
|
|
552
611
|
// The prefix matches exactly: LIKE ignored case and read "_" and "%" as wildcards.
|
|
553
|
-
list: store.query<
|
|
612
|
+
list: store.query<
|
|
613
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
614
|
+
[string, string, string, number, number]
|
|
615
|
+
>(
|
|
554
616
|
"SELECT key, expires_at, metadata FROM kv WHERE ns = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 " +
|
|
555
617
|
"AND (expires_at IS NULL OR expires_at > ?4) ORDER BY key LIMIT ?5",
|
|
556
618
|
),
|
|
@@ -566,7 +628,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
566
628
|
* seconds) -> ms epoch, or `null` for no expiry. */
|
|
567
629
|
const kvExpiresAt = (msg: Frame): number | null => {
|
|
568
630
|
if (isNumber(msg.expirationTtl)) {
|
|
569
|
-
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
631
|
+
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
632
|
+
throw new Error(`expirationTtl must be at least ${MIN_KV_TTL_SECONDS} seconds`);
|
|
570
633
|
return Date.now() + msg.expirationTtl * 1000;
|
|
571
634
|
}
|
|
572
635
|
if (isNumber(msg.expiration)) return msg.expiration * 1000;
|
|
@@ -761,7 +824,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
761
824
|
});
|
|
762
825
|
};
|
|
763
826
|
|
|
764
|
-
const validateCompletedParts = (
|
|
827
|
+
const validateCompletedParts = (
|
|
828
|
+
stored: R2PartRow[],
|
|
829
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
830
|
+
): void => {
|
|
765
831
|
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
766
832
|
for (let index = 0; index < stored.length; index++) {
|
|
767
833
|
const actual = stored[index];
|
|
@@ -862,7 +928,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
862
928
|
const path = join(directory, name);
|
|
863
929
|
try {
|
|
864
930
|
const tokenValue = name.slice(0, 24);
|
|
865
|
-
const ticket = store
|
|
931
|
+
const ticket = store
|
|
932
|
+
.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?")
|
|
933
|
+
.get(tokenValue);
|
|
866
934
|
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
867
935
|
} catch (error) {
|
|
868
936
|
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
@@ -884,7 +952,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
884
952
|
}
|
|
885
953
|
const start = Number(match[1]);
|
|
886
954
|
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
887
|
-
if (
|
|
955
|
+
if (
|
|
956
|
+
!Number.isSafeInteger(start) ||
|
|
957
|
+
!Number.isSafeInteger(requestedEnd) ||
|
|
958
|
+
start < 0 ||
|
|
959
|
+
start >= size ||
|
|
960
|
+
requestedEnd < start
|
|
961
|
+
)
|
|
962
|
+
return null;
|
|
888
963
|
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
889
964
|
};
|
|
890
965
|
|
|
@@ -911,11 +986,16 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
911
986
|
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
912
987
|
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
913
988
|
}
|
|
914
|
-
if (
|
|
989
|
+
if (
|
|
990
|
+
(ticket.method === "upload" && method !== "PUT") ||
|
|
991
|
+
(ticket.method === "download" && method !== "GET" && method !== "HEAD")
|
|
992
|
+
) {
|
|
915
993
|
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
916
994
|
}
|
|
917
995
|
if (ticket.method === "download") {
|
|
918
|
-
const row = store
|
|
996
|
+
const row = store
|
|
997
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
998
|
+
.get(bucket, ticket.key);
|
|
919
999
|
if (!row) return new Response("not found", { status: 404 });
|
|
920
1000
|
const etag = `"${row.etag}"`;
|
|
921
1001
|
const noneMatch = request.headers.get("if-none-match");
|
|
@@ -947,7 +1027,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
947
1027
|
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
948
1028
|
return new Response("invalid content length", { status: 400 });
|
|
949
1029
|
}
|
|
950
|
-
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1030
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1031
|
+
return new Response("upload too large", { status: 413 });
|
|
951
1032
|
const directory = r2BlobDir(store);
|
|
952
1033
|
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
953
1034
|
if (activeUploads >= maxActiveUploads) {
|
|
@@ -1000,7 +1081,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1000
1081
|
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
1001
1082
|
renameSync(temporary, destination);
|
|
1002
1083
|
const previous = store
|
|
1003
|
-
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1084
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1085
|
+
"SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?",
|
|
1086
|
+
)
|
|
1004
1087
|
.get(bucket, ticket.key);
|
|
1005
1088
|
const row: R2Row = {
|
|
1006
1089
|
key: ticket.key,
|
|
@@ -1059,7 +1142,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1059
1142
|
try {
|
|
1060
1143
|
for (const part of parts) {
|
|
1061
1144
|
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1062
|
-
if (!bytes || bytes.byteLength !== part.size)
|
|
1145
|
+
if (!bytes || bytes.byteLength !== part.size)
|
|
1146
|
+
throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1063
1147
|
appendFileSync(temporary, bytes);
|
|
1064
1148
|
hash.update(bytes);
|
|
1065
1149
|
size += bytes.byteLength;
|
|
@@ -1105,7 +1189,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1105
1189
|
// group counts as one entry). Same rules as the embedded transport's __sbR2ListPage.
|
|
1106
1190
|
// ponytail: with a delimiter every key after the cursor is read; an index on the
|
|
1107
1191
|
// delimited prefix if buckets grow past ~100k keys.
|
|
1108
|
-
const r2ListPage = (
|
|
1192
|
+
const r2ListPage = (
|
|
1193
|
+
store: Database,
|
|
1194
|
+
bucket: string,
|
|
1195
|
+
prefix: string,
|
|
1196
|
+
cursor: string,
|
|
1197
|
+
limit: number,
|
|
1198
|
+
delimiter: string,
|
|
1199
|
+
) => {
|
|
1109
1200
|
const rows = store
|
|
1110
1201
|
.query<R2Row, [string, string, string]>(
|
|
1111
1202
|
"SELECT * FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 ORDER BY key" +
|
|
@@ -1230,7 +1321,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1230
1321
|
}
|
|
1231
1322
|
}
|
|
1232
1323
|
|
|
1233
|
-
async function proxyFetch(
|
|
1324
|
+
async function proxyFetch(
|
|
1325
|
+
msg: Frame,
|
|
1326
|
+
signal?: AbortSignal,
|
|
1327
|
+
emit?: FrameEmitter,
|
|
1328
|
+
body?: ReadableStream<Uint8Array>,
|
|
1329
|
+
): Promise<Upstream> {
|
|
1234
1330
|
let url: URL;
|
|
1235
1331
|
try {
|
|
1236
1332
|
url = new URL(str(msg.url));
|
|
@@ -1270,7 +1366,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1270
1366
|
* minus the trip through the network (and therefore minus TLS and the
|
|
1271
1367
|
* default-deny egress rules, which is why this is not `fetch`).
|
|
1272
1368
|
*/
|
|
1273
|
-
async function serviceFetch(
|
|
1369
|
+
async function serviceFetch(
|
|
1370
|
+
msg: Frame,
|
|
1371
|
+
signal?: AbortSignal,
|
|
1372
|
+
emit?: FrameEmitter,
|
|
1373
|
+
body?: ReadableStream<Uint8Array>,
|
|
1374
|
+
): Promise<Upstream> {
|
|
1274
1375
|
const binding = str(msg.binding);
|
|
1275
1376
|
const declared = bindings.services.find((entry) => entry.binding === binding);
|
|
1276
1377
|
if (!declared) throw new Error(`service not bound: ${binding}`);
|
|
@@ -1360,7 +1461,13 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1360
1461
|
// last key), with expirations and metadata. Without one: up to 1,000 keys, as before.
|
|
1361
1462
|
const paged = isNumber(msg.limit);
|
|
1362
1463
|
const limit = paged ? Math.max(0, Math.floor(Number(msg.limit))) : 1000;
|
|
1363
|
-
const rows = kvStmts(store).list.all(
|
|
1464
|
+
const rows = kvStmts(store).list.all(
|
|
1465
|
+
part,
|
|
1466
|
+
str(msg.prefix),
|
|
1467
|
+
paged ? str(msg.cursor) : "",
|
|
1468
|
+
Date.now(),
|
|
1469
|
+
paged ? limit + 1 : limit,
|
|
1470
|
+
);
|
|
1364
1471
|
const page = rows.slice(0, limit);
|
|
1365
1472
|
const complete = !paged || rows.length <= limit;
|
|
1366
1473
|
return {
|
|
@@ -1379,17 +1486,22 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1379
1486
|
const name = str(msg.name) || "default";
|
|
1380
1487
|
const key = str(msg.key);
|
|
1381
1488
|
const row = db
|
|
1382
|
-
.query<
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
>("SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?")
|
|
1489
|
+
.query<{ status: number; headers_json: string; body: string; expires_at: number | null }, [string, string]>(
|
|
1490
|
+
"SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?",
|
|
1491
|
+
)
|
|
1386
1492
|
.get(name, key);
|
|
1387
1493
|
if (!row) return { ok: true, found: false };
|
|
1388
1494
|
if (row.expires_at != null && row.expires_at <= Date.now()) {
|
|
1389
1495
|
db.query("DELETE FROM cache WHERE name = ? AND cache_key = ?").run(name, key);
|
|
1390
1496
|
return { ok: true, found: false };
|
|
1391
1497
|
}
|
|
1392
|
-
return {
|
|
1498
|
+
return {
|
|
1499
|
+
ok: true,
|
|
1500
|
+
found: true,
|
|
1501
|
+
status: row.status,
|
|
1502
|
+
headers: jsonObject(parseJsonValue(row.headers_json)) ?? {},
|
|
1503
|
+
body: row.body,
|
|
1504
|
+
};
|
|
1393
1505
|
}
|
|
1394
1506
|
case "cache.put": {
|
|
1395
1507
|
const name = str(msg.name) || "default";
|
|
@@ -1503,9 +1615,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1503
1615
|
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1504
1616
|
const expires = Date.now() + expiresIn * 1000;
|
|
1505
1617
|
const maxBytes =
|
|
1506
|
-
method === "upload"
|
|
1507
|
-
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1508
|
-
: 0;
|
|
1618
|
+
method === "upload" ? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit()) : 0;
|
|
1509
1619
|
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1510
1620
|
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1511
1621
|
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
@@ -1565,11 +1675,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1565
1675
|
case "r2.head": {
|
|
1566
1676
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1567
1677
|
const key = str(msg.key);
|
|
1568
|
-
const row = store
|
|
1678
|
+
const row = store
|
|
1679
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1680
|
+
.get(bucket, key);
|
|
1569
1681
|
if (!row) return { ok: true, found: false };
|
|
1570
|
-
const body =
|
|
1571
|
-
|
|
1572
|
-
|
|
1682
|
+
const body =
|
|
1683
|
+
msg.op === "r2.get"
|
|
1684
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1685
|
+
: undefined;
|
|
1573
1686
|
return { ok: true, found: true, object: r2Row(row), body };
|
|
1574
1687
|
}
|
|
1575
1688
|
case "r2.delete": {
|
|
@@ -1585,7 +1698,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1585
1698
|
case "r2.md5": {
|
|
1586
1699
|
// Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
|
|
1587
1700
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1588
|
-
store
|
|
1701
|
+
store
|
|
1702
|
+
.query("UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4")
|
|
1703
|
+
.run(str(msg.md5), bucket, str(msg.key), str(msg.etag));
|
|
1589
1704
|
return { ok: true };
|
|
1590
1705
|
}
|
|
1591
1706
|
case "r2.list": {
|
|
@@ -1673,10 +1788,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1673
1788
|
return row ? { ok: true, found: true, value: row.value } : { ok: true, found: false };
|
|
1674
1789
|
}
|
|
1675
1790
|
case "do.storage.put":
|
|
1676
|
-
doDb
|
|
1677
|
-
|
|
1678
|
-
"
|
|
1679
|
-
|
|
1791
|
+
doDb
|
|
1792
|
+
.query(
|
|
1793
|
+
"INSERT INTO do_storage (cls, id, key, value) VALUES (?1,?2,?3,?4) " +
|
|
1794
|
+
"ON CONFLICT (cls, id, key) DO UPDATE SET value = ?4",
|
|
1795
|
+
)
|
|
1796
|
+
.run(requireDoClass(msg.cls), str(msg.id), str(msg.key), str(msg.value));
|
|
1680
1797
|
return { ok: true };
|
|
1681
1798
|
case "do.storage.delete": {
|
|
1682
1799
|
const r = doDb
|
|
@@ -1690,11 +1807,20 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1690
1807
|
case "do.storage.list": {
|
|
1691
1808
|
const cls = requireDoClass(msg.cls);
|
|
1692
1809
|
const limit = Math.min(Math.max(Number(msg.limit) || 1000, 1), 10000);
|
|
1810
|
+
// start, startAfter, end and reverse come from the runtime's 2026-10-28 storage.
|
|
1811
|
+
// The prefix match is exact: LIKE also matched _ and % as wildcards, case-insensitively.
|
|
1812
|
+
const bound = (v: JsonValue | undefined): string | null => (v == null ? null : str(v));
|
|
1693
1813
|
const rows = doDb
|
|
1694
|
-
.query<
|
|
1695
|
-
|
|
1814
|
+
.query<
|
|
1815
|
+
{ key: string; value: string },
|
|
1816
|
+
[string, string, string, number, string | null, string | null, string | null]
|
|
1817
|
+
>(
|
|
1818
|
+
"SELECT key, value FROM do_storage WHERE cls = ?1 AND id = ?2 AND substr(key, 1, length(?3)) = ?3" +
|
|
1819
|
+
" AND (?5 IS NULL OR key >= ?5) AND (?6 IS NULL OR key > ?6) AND (?7 IS NULL OR key < ?7)" +
|
|
1820
|
+
(msg.reverse === true ? " ORDER BY key DESC" : " ORDER BY key") +
|
|
1821
|
+
" LIMIT ?4",
|
|
1696
1822
|
)
|
|
1697
|
-
.all(cls, str(msg.id), str(msg.prefix), limit);
|
|
1823
|
+
.all(cls, str(msg.id), str(msg.prefix), limit, bound(msg.start), bound(msg.startAfter), bound(msg.end));
|
|
1698
1824
|
return { ok: true, entries: rows.map((r) => [r.key, r.value]) };
|
|
1699
1825
|
}
|
|
1700
1826
|
|
|
@@ -1702,17 +1828,23 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1702
1828
|
// for that object, and attempts resets because this is a fresh schedule.
|
|
1703
1829
|
case "do.alarm.set": {
|
|
1704
1830
|
const cls = requireDoClass(msg.cls);
|
|
1705
|
-
doDb
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1831
|
+
doDb
|
|
1832
|
+
.transaction(() => {
|
|
1833
|
+
doDb
|
|
1834
|
+
.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=? AND expires<=?")
|
|
1835
|
+
.run(cls, str(msg.id), Date.now());
|
|
1836
|
+
doDb
|
|
1837
|
+
.query(
|
|
1838
|
+
"INSERT INTO do_alarm (cls,id,at,attempts) VALUES (?1,?2,?3,0) ON CONFLICT (cls,id) DO UPDATE SET at=?3,attempts=0",
|
|
1839
|
+
)
|
|
1840
|
+
.run(cls, str(msg.id), Math.trunc(Number(msg.at) || 0));
|
|
1841
|
+
})
|
|
1842
|
+
.immediate();
|
|
1843
|
+
return { ok: true };
|
|
1710
1844
|
}
|
|
1711
1845
|
case "do.alarm.get": {
|
|
1712
1846
|
const row = doDb
|
|
1713
|
-
.query<{ at: number }, [string, string]>(
|
|
1714
|
-
"SELECT at FROM do_alarm WHERE cls = ? AND id = ?",
|
|
1715
|
-
)
|
|
1847
|
+
.query<{ at: number }, [string, string]>("SELECT at FROM do_alarm WHERE cls = ? AND id = ?")
|
|
1716
1848
|
.get(requireDoClass(msg.cls), str(msg.id));
|
|
1717
1849
|
const retry = row
|
|
1718
1850
|
? null
|
|
@@ -1725,12 +1857,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1725
1857
|
}
|
|
1726
1858
|
case "do.alarm.delete": {
|
|
1727
1859
|
const cls = requireDoClass(msg.cls);
|
|
1728
|
-
const deleted = doDb
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1860
|
+
const deleted = doDb
|
|
1861
|
+
.transaction(() => {
|
|
1862
|
+
const pending = doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1863
|
+
const delivery = doDb.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1864
|
+
return pending.changes > 0 || delivery.changes > 0;
|
|
1865
|
+
})
|
|
1866
|
+
.immediate();
|
|
1867
|
+
return { ok: true, deleted };
|
|
1734
1868
|
}
|
|
1735
1869
|
|
|
1736
1870
|
case "assets.get": {
|
|
@@ -1830,7 +1964,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1830
1964
|
if (msg.op === "r2.get") {
|
|
1831
1965
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1832
1966
|
const key = str(msg.key);
|
|
1833
|
-
const row = store
|
|
1967
|
+
const row = store
|
|
1968
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1969
|
+
.get(bucket, key);
|
|
1834
1970
|
if (!row) return { reply: { ok: true, found: false } };
|
|
1835
1971
|
return {
|
|
1836
1972
|
reply: { ok: true, found: true, object: r2Row(row) },
|
|
@@ -1861,7 +1997,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1861
1997
|
} else if (!hit && assetManifest?.notFound === "404-page") {
|
|
1862
1998
|
hit = readAsset("/404.html");
|
|
1863
1999
|
}
|
|
1864
|
-
if (!hit)
|
|
2000
|
+
if (!hit)
|
|
2001
|
+
return { reply: { ok: true, found: false, status: 404, rules }, bytes: new TextEncoder().encode("Not Found") };
|
|
1865
2002
|
return {
|
|
1866
2003
|
reply: { ok: true, found, status, type: hit.type, hash: hit.hash, rules },
|
|
1867
2004
|
bytes: hit.body,
|
|
@@ -1872,7 +2009,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1872
2009
|
}
|
|
1873
2010
|
|
|
1874
2011
|
/** One wire frame in, one out. See `handleFrame` on the Broker type. */
|
|
1875
|
-
async function handleFrame(
|
|
2012
|
+
async function handleFrame(
|
|
2013
|
+
payload: Buffer,
|
|
2014
|
+
signal?: AbortSignal,
|
|
2015
|
+
emit?: FrameEmitter,
|
|
2016
|
+
body?: ReadableStream<Uint8Array>,
|
|
2017
|
+
): Promise<Buffer> {
|
|
1876
2018
|
if (payload.length === 0 || payload[0] !== FRAME_V1) {
|
|
1877
2019
|
return encodeFrame(await handlePayload(payload.toString("utf8")));
|
|
1878
2020
|
}
|
|
@@ -1983,14 +2125,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1983
2125
|
|
|
1984
2126
|
const timeoutMs = opts.deliveryTimeoutMs ?? 25_000;
|
|
1985
2127
|
const leaseMs = opts.deliveryLeaseMs ?? 30_000;
|
|
1986
|
-
if (!(timeoutMs > 0 && leaseMs > timeoutMs))
|
|
1987
|
-
throw new Error("delivery lease must exceed its timeout");
|
|
2128
|
+
if (!(timeoutMs > 0 && leaseMs > timeoutMs)) throw new Error("delivery lease must exceed its timeout");
|
|
1988
2129
|
let closed = false;
|
|
1989
2130
|
const deliveries = new Set<AbortController>();
|
|
1990
|
-
async function deliverTrigger(
|
|
1991
|
-
kind: "scheduled" | "queue" | "alarm",
|
|
1992
|
-
body: JsonObject,
|
|
1993
|
-
): Promise<Response | null> {
|
|
2131
|
+
async function deliverTrigger(kind: "scheduled" | "queue" | "alarm", body: JsonObject): Promise<Response | null> {
|
|
1994
2132
|
if (!opts.sproutUrl || !dispatchEnabled() || closed) return null;
|
|
1995
2133
|
const abort = new AbortController();
|
|
1996
2134
|
deliveries.add(abort);
|
|
@@ -2007,21 +2145,30 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2007
2145
|
},
|
|
2008
2146
|
body: JSON.stringify(body),
|
|
2009
2147
|
}).then(async (res) => {
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2148
|
+
if (kind !== "queue") {
|
|
2149
|
+
await res.body?.cancel();
|
|
2150
|
+
return new Response(null, { status: res.status });
|
|
2151
|
+
}
|
|
2152
|
+
const chunks: Uint8Array[] = [];
|
|
2153
|
+
let size = 0;
|
|
2154
|
+
const reader = res.body?.getReader();
|
|
2155
|
+
try {
|
|
2156
|
+
if (reader)
|
|
2157
|
+
for (;;) {
|
|
2016
2158
|
const part = await reader.read();
|
|
2017
2159
|
if (part.done) break;
|
|
2018
2160
|
size += part.value.byteLength;
|
|
2019
|
-
if (size > 65_536) {
|
|
2161
|
+
if (size > 65_536) {
|
|
2162
|
+
await reader.cancel();
|
|
2163
|
+
throw new Error("delivery acknowledgement exceeds 64 KiB");
|
|
2164
|
+
}
|
|
2020
2165
|
chunks.push(part.value);
|
|
2021
2166
|
}
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
}
|
|
2167
|
+
} finally {
|
|
2168
|
+
reader?.releaseLock();
|
|
2169
|
+
}
|
|
2170
|
+
return new Response(size ? Buffer.concat(chunks) : null, { status: res.status, headers: res.headers });
|
|
2171
|
+
}),
|
|
2025
2172
|
new Promise<never>((_, reject) =>
|
|
2026
2173
|
abort.signal.addEventListener("abort", () => reject(new Error("delivery interrupted")), {
|
|
2027
2174
|
once: true,
|
|
@@ -2098,9 +2245,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2098
2245
|
for (const id of ack)
|
|
2099
2246
|
if (!retrySet.has(id))
|
|
2100
2247
|
store
|
|
2101
|
-
.query(
|
|
2102
|
-
"DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?",
|
|
2103
|
-
)
|
|
2248
|
+
.query("DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?")
|
|
2104
2249
|
.run(part, id, claim, Date.now());
|
|
2105
2250
|
for (const id of retry)
|
|
2106
2251
|
store
|
|
@@ -2120,7 +2265,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2120
2265
|
const now = Date.now();
|
|
2121
2266
|
const claimed = doDb
|
|
2122
2267
|
.transaction(() => {
|
|
2123
|
-
doDb
|
|
2268
|
+
doDb
|
|
2269
|
+
.query(
|
|
2270
|
+
"DELETE FROM do_alarm_delivery WHERE expires<=? AND attempts>=5 AND EXISTS (SELECT 1 FROM do_alarm a WHERE a.cls=do_alarm_delivery.cls AND a.id=do_alarm_delivery.id)",
|
|
2271
|
+
)
|
|
2272
|
+
.run(now);
|
|
2124
2273
|
const due = doDb
|
|
2125
2274
|
.query<{ cls: string; id: string; at: number; attempts: number }, [number]>(
|
|
2126
2275
|
"SELECT cls,id,at,attempts FROM do_alarm WHERE at <= ? AND NOT EXISTS (SELECT 1 FROM do_alarm_delivery d WHERE d.cls=do_alarm.cls AND d.id=do_alarm.id) ORDER BY at LIMIT 10",
|
|
@@ -2128,9 +2277,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2128
2277
|
.all(now)
|
|
2129
2278
|
.filter((r) => bindings.do.some((b) => b.className === r.cls));
|
|
2130
2279
|
for (const r of due) {
|
|
2131
|
-
doDb
|
|
2132
|
-
.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)")
|
|
2133
|
-
.run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2280
|
+
doDb.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)").run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2134
2281
|
doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=? AND at=?").run(r.cls, r.id, r.at);
|
|
2135
2282
|
}
|
|
2136
2283
|
const expired = doDb
|
|
@@ -2142,9 +2289,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2142
2289
|
return expired.map((r) => {
|
|
2143
2290
|
const claim = randomBytes(24).toString("hex");
|
|
2144
2291
|
doDb
|
|
2145
|
-
.query(
|
|
2146
|
-
"UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?",
|
|
2147
|
-
)
|
|
2292
|
+
.query("UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?")
|
|
2148
2293
|
.run(claim, now + leaseMs, r.cls, r.id);
|
|
2149
2294
|
return { ...r, claim };
|
|
2150
2295
|
});
|
|
@@ -2159,9 +2304,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2159
2304
|
.run(r.cls, r.id, r.claim, Date.now());
|
|
2160
2305
|
else
|
|
2161
2306
|
doDb
|
|
2162
|
-
.query(
|
|
2163
|
-
"UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?",
|
|
2164
|
-
)
|
|
2307
|
+
.query("UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?")
|
|
2165
2308
|
.run(Date.now() + 5_000, r.cls, r.id, r.claim, Date.now());
|
|
2166
2309
|
});
|
|
2167
2310
|
}
|
|
@@ -2179,8 +2322,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2179
2322
|
if (stamp === lastTick) return; // once per minute
|
|
2180
2323
|
lastTick = stamp;
|
|
2181
2324
|
for (const expr of bindings.crons) {
|
|
2182
|
-
if (cronMatches(expr, now))
|
|
2183
|
-
void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2325
|
+
if (cronMatches(expr, now)) void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2184
2326
|
}
|
|
2185
2327
|
}, 15_000),
|
|
2186
2328
|
);
|
|
@@ -2344,7 +2486,10 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2344
2486
|
const state = socket.data;
|
|
2345
2487
|
if (state.closed || !state.output) return;
|
|
2346
2488
|
const accepted = socket.write(state.output.subarray(state.offset));
|
|
2347
|
-
if (accepted < 0) {
|
|
2489
|
+
if (accepted < 0) {
|
|
2490
|
+
socket.end();
|
|
2491
|
+
return;
|
|
2492
|
+
}
|
|
2348
2493
|
state.offset += accepted;
|
|
2349
2494
|
if (state.offset === state.output.length) {
|
|
2350
2495
|
state.output = null;
|
|
@@ -2398,47 +2543,56 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2398
2543
|
state.size += chunk.length;
|
|
2399
2544
|
if (state.processing) return;
|
|
2400
2545
|
state.processing = true;
|
|
2401
|
-
try {
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2546
|
+
try {
|
|
2547
|
+
for (;;) {
|
|
2548
|
+
if (state.closed) return;
|
|
2549
|
+
if (state.size < 4) return;
|
|
2550
|
+
if (state.chunks.length > 1) {
|
|
2551
|
+
state.chunks = [Buffer.concat(state.chunks, state.size)];
|
|
2552
|
+
}
|
|
2553
|
+
const buf = state.chunks[0];
|
|
2554
|
+
const len = buf.readUInt32LE(0);
|
|
2555
|
+
if (len > MAX_FRAME) {
|
|
2556
|
+
await send(socket, encodeFrame({ ok: false, error: "frame too large" }));
|
|
2557
|
+
socket.end();
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
if (state.size < 4 + len) return;
|
|
2561
|
+
// Kept as bytes: a v1 frame carries binary after its JSON, and
|
|
2562
|
+
// decoding the whole payload as utf8 would corrupt it.
|
|
2563
|
+
const payload = buf.subarray(4, 4 + len);
|
|
2564
|
+
const rest = buf.subarray(4 + len);
|
|
2565
|
+
state.chunks = rest.length ? [Buffer.from(rest)] : [];
|
|
2566
|
+
state.size = rest.length;
|
|
2567
|
+
// A frame of a streamed request body: to its request's stream.
|
|
2568
|
+
if (payload.length > 0 && payload[0] === FRAME_V1_MORE) {
|
|
2569
|
+
state.sink?.frame(payload);
|
|
2570
|
+
continue;
|
|
2571
|
+
}
|
|
2572
|
+
// A request whose body follows in frames: it runs while this loop feeds them.
|
|
2573
|
+
if (bodyStreamHead(payload)) {
|
|
2574
|
+
const sink = bodySink(socket);
|
|
2575
|
+
state.sink = sink;
|
|
2576
|
+
void (async () => {
|
|
2577
|
+
const reply = await broker.handleFrame(
|
|
2578
|
+
payload,
|
|
2579
|
+
state.abort.signal,
|
|
2580
|
+
(frame) => send(socket, frame),
|
|
2581
|
+
sink.stream,
|
|
2582
|
+
);
|
|
2583
|
+
await send(socket, reply);
|
|
2584
|
+
// Body frames the sprout still sends (the upstream answered early) are dropped.
|
|
2585
|
+
sink.fail("the request already ended");
|
|
2586
|
+
})();
|
|
2587
|
+
continue;
|
|
2588
|
+
}
|
|
2589
|
+
// Bun does not buffer unwritten bytes. Serialize replies and retain
|
|
2590
|
+
// the unsent suffix until drain, including binary R2 responses.
|
|
2591
|
+
await send(socket, await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame)));
|
|
2437
2592
|
}
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
} } finally { state.processing = false; }
|
|
2593
|
+
} finally {
|
|
2594
|
+
state.processing = false;
|
|
2595
|
+
}
|
|
2442
2596
|
},
|
|
2443
2597
|
},
|
|
2444
2598
|
});
|
|
@@ -2449,10 +2603,15 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2449
2603
|
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
2450
2604
|
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
2451
2605
|
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
2452
|
-
const server = Bun.serve({
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2606
|
+
const server = Bun.serve({
|
|
2607
|
+
hostname,
|
|
2608
|
+
port,
|
|
2609
|
+
maxRequestBodySize,
|
|
2610
|
+
fetch(request, server) {
|
|
2611
|
+
server.timeout(request, 255);
|
|
2612
|
+
return broker.transfer(request);
|
|
2613
|
+
},
|
|
2614
|
+
});
|
|
2456
2615
|
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
2457
2616
|
}
|
|
2458
2617
|
|
|
@@ -2498,9 +2657,7 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2498
2657
|
const parsed = jsonObject(parseJsonValue(values.services));
|
|
2499
2658
|
if (parsed) {
|
|
2500
2659
|
services = Object.fromEntries(
|
|
2501
|
-
Object.entries(parsed).flatMap(([binding, host]) =>
|
|
2502
|
-
isString(host) ? [[binding, host] as const] : [],
|
|
2503
|
-
),
|
|
2660
|
+
Object.entries(parsed).flatMap(([binding, host]) => (isString(host) ? [[binding, host] as const] : [])),
|
|
2504
2661
|
);
|
|
2505
2662
|
}
|
|
2506
2663
|
} catch {
|
|
@@ -2518,13 +2675,9 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2518
2675
|
resourceDir: values["resource-dir"],
|
|
2519
2676
|
doDb: values["do-db"],
|
|
2520
2677
|
ownerId: values["owner-id"],
|
|
2521
|
-
r2QuotaBytes:
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
? String(values["r2-resource-ids"]).split(",").filter(Boolean)
|
|
2525
|
-
: undefined,
|
|
2526
|
-
r2MinFreeBytes:
|
|
2527
|
-
Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2678
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2679
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2680
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2528
2681
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
2529
2682
|
bindings,
|
|
2530
2683
|
secrets,
|