@sproutboat/wire 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/package.json +2 -2
- package/src/broker.test.ts +384 -97
- package/src/broker.ts +417 -160
package/src/broker.ts
CHANGED
|
@@ -21,7 +21,18 @@ import { createD1Store } from "./d1";
|
|
|
21
21
|
import type { D1Pool } from "./d1-pool";
|
|
22
22
|
import { Database, type Statement } from "bun:sqlite";
|
|
23
23
|
import { createHash, randomBytes } from "node:crypto";
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
appendFileSync,
|
|
26
|
+
existsSync,
|
|
27
|
+
mkdirSync,
|
|
28
|
+
readFileSync,
|
|
29
|
+
readdirSync,
|
|
30
|
+
renameSync,
|
|
31
|
+
rmSync,
|
|
32
|
+
statfsSync,
|
|
33
|
+
statSync,
|
|
34
|
+
writeFileSync,
|
|
35
|
+
} from "node:fs";
|
|
25
36
|
import { open } from "node:fs/promises";
|
|
26
37
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
27
38
|
import { parseArgs } from "node:util";
|
|
@@ -196,9 +207,16 @@ export type Broker = {
|
|
|
196
207
|
* `service.fetch` in flight then stops its upstream request too. With `emit`, a
|
|
197
208
|
* `fetch` or `service.fetch` that asks to stream (`stream: true`) sends its head and
|
|
198
209
|
* each body chunk through it as they arrive (FRAME_V1_MORE frames); the returned
|
|
199
|
-
* frame ends the response.
|
|
210
|
+
* frame ends the response. With `body`, a `fetch` or `service.fetch` whose head asked
|
|
211
|
+
* for it (`bodyStream: true`) sends that stream as its request body: the listener feeds
|
|
212
|
+
* it from the FRAME_V1_MORE frames that follow the head on the same connection.
|
|
200
213
|
*/
|
|
201
|
-
handleFrame(
|
|
214
|
+
handleFrame(
|
|
215
|
+
payload: Buffer,
|
|
216
|
+
signal?: AbortSignal,
|
|
217
|
+
emit?: FrameEmitter,
|
|
218
|
+
body?: ReadableStream<Uint8Array>,
|
|
219
|
+
): Promise<Buffer>;
|
|
202
220
|
/** Serve one reserved direct R2 upload or download request. */
|
|
203
221
|
transfer(request: Request): Promise<Response>;
|
|
204
222
|
close(): void;
|
|
@@ -298,10 +316,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
298
316
|
"expires_at INTEGER, PRIMARY KEY (ns, key))",
|
|
299
317
|
);
|
|
300
318
|
const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
|
|
301
|
-
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
319
|
+
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
320
|
+
conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
|
|
302
321
|
// #233 — 1 when the value was put as bytes (stored one char per byte), so get()
|
|
303
322
|
// can hand back exact bytes or decoded text depending on the requested type.
|
|
304
|
-
if (!kvColumns.some((column) => column.name === "binary"))
|
|
323
|
+
if (!kvColumns.some((column) => column.name === "binary"))
|
|
324
|
+
conn.exec("ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0");
|
|
305
325
|
// A key's metadata (put(key, value, { metadata })), as JSON text.
|
|
306
326
|
if (!kvColumns.some((column) => column.name === "metadata")) conn.exec("ALTER TABLE kv ADD COLUMN metadata TEXT");
|
|
307
327
|
// #56 — no `body` column: an object's bytes live in their own file under
|
|
@@ -342,7 +362,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
342
362
|
// Every queue poll filters one partition's live rows and takes its earliest
|
|
343
363
|
// visible batch. This additive index also upgrades existing resource files.
|
|
344
364
|
conn.exec("CREATE INDEX IF NOT EXISTS mq_due ON mq (queue, dead, visible_at)");
|
|
345
|
-
if (
|
|
365
|
+
if (
|
|
366
|
+
!conn
|
|
367
|
+
.query<{ name: string }, []>("PRAGMA table_info(mq)")
|
|
368
|
+
.all()
|
|
369
|
+
.some((c) => c.name === "claim_token")
|
|
370
|
+
)
|
|
346
371
|
conn.exec("ALTER TABLE mq ADD COLUMN claim_token TEXT NOT NULL DEFAULT ''");
|
|
347
372
|
return conn;
|
|
348
373
|
};
|
|
@@ -371,7 +396,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
371
396
|
);
|
|
372
397
|
// Alarm polling reads the earliest due objects across classes.
|
|
373
398
|
doDb.exec("CREATE INDEX IF NOT EXISTS do_alarm_due ON do_alarm (at)");
|
|
374
|
-
doDb.exec(
|
|
399
|
+
doDb.exec(
|
|
400
|
+
"CREATE TABLE IF NOT EXISTS do_alarm_delivery (cls TEXT NOT NULL, id TEXT NOT NULL, at INTEGER NOT NULL, attempts INTEGER NOT NULL, token TEXT NOT NULL, expires INTEGER NOT NULL, PRIMARY KEY(cls,id))",
|
|
401
|
+
);
|
|
375
402
|
// #69 — fixed-window rate-limiter counters, per binding name + key.
|
|
376
403
|
db.exec(
|
|
377
404
|
"CREATE TABLE IF NOT EXISTS ratelimit (name TEXT NOT NULL, key TEXT NOT NULL, window_start INTEGER NOT NULL, " +
|
|
@@ -405,9 +432,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
405
432
|
// One ledger for every account, shared by the brokers of all its deployments.
|
|
406
433
|
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
407
434
|
// requests cannot each independently decide that the same remaining bytes fit.
|
|
408
|
-
const quotaDb =
|
|
409
|
-
|
|
410
|
-
|
|
435
|
+
const quotaDb =
|
|
436
|
+
quotaOwner && quotaDirectory && quotaBytes
|
|
437
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
438
|
+
: null;
|
|
411
439
|
if (quotaDb) {
|
|
412
440
|
const owner = quotaOwner!;
|
|
413
441
|
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
@@ -420,7 +448,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
420
448
|
quotaDb.exec(
|
|
421
449
|
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
422
450
|
);
|
|
423
|
-
const existing = quotaDb
|
|
451
|
+
const existing = quotaDb
|
|
452
|
+
.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?")
|
|
453
|
+
.get(owner);
|
|
424
454
|
if (!existing) {
|
|
425
455
|
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
426
456
|
}
|
|
@@ -434,9 +464,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
434
464
|
// Open through resourceDb so older resource files gain the multipart
|
|
435
465
|
// tables before the aggregate is queried.
|
|
436
466
|
const store = resourceDb(id);
|
|
437
|
-
used +=
|
|
438
|
-
|
|
439
|
-
|
|
467
|
+
used +=
|
|
468
|
+
store
|
|
469
|
+
.query<{ bytes: number }, []>(
|
|
470
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
471
|
+
)
|
|
472
|
+
.get()?.bytes ?? 0;
|
|
440
473
|
}
|
|
441
474
|
return used;
|
|
442
475
|
};
|
|
@@ -451,9 +484,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
451
484
|
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
452
485
|
if (!quotaDb || !quotaOwner) return;
|
|
453
486
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
454
|
-
const reserved =
|
|
455
|
-
|
|
456
|
-
|
|
487
|
+
const reserved =
|
|
488
|
+
quotaDb
|
|
489
|
+
.query<{ bytes: number }, [string]>(
|
|
490
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
491
|
+
)
|
|
492
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
457
493
|
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
458
494
|
};
|
|
459
495
|
|
|
@@ -463,10 +499,15 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
463
499
|
try {
|
|
464
500
|
releaseExpiredR2Reservations();
|
|
465
501
|
const used = committedR2Bytes();
|
|
466
|
-
const reserved =
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
502
|
+
const reserved =
|
|
503
|
+
quotaDb
|
|
504
|
+
.query<{ bytes: number }, [string]>(
|
|
505
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
506
|
+
)
|
|
507
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
508
|
+
quotaDb
|
|
509
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
510
|
+
.run(used, reserved, quotaOwner);
|
|
470
511
|
if (used + reserved + bytes > quotaBytes) {
|
|
471
512
|
quotaDb.exec("ROLLBACK");
|
|
472
513
|
return "quota";
|
|
@@ -476,8 +517,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
476
517
|
quotaDb.exec("ROLLBACK");
|
|
477
518
|
return "disk";
|
|
478
519
|
}
|
|
479
|
-
quotaDb
|
|
480
|
-
|
|
520
|
+
quotaDb
|
|
521
|
+
.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)")
|
|
522
|
+
.run(token, quotaOwner, bytes, expires);
|
|
523
|
+
quotaDb
|
|
524
|
+
.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2")
|
|
525
|
+
.run(reserved + bytes, quotaOwner);
|
|
481
526
|
quotaDb.exec("COMMIT");
|
|
482
527
|
return "ok";
|
|
483
528
|
} catch (error) {
|
|
@@ -490,17 +535,24 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
490
535
|
if (!quotaDb || !quotaOwner) return;
|
|
491
536
|
quotaDb.exec("BEGIN IMMEDIATE");
|
|
492
537
|
try {
|
|
493
|
-
const reservation = quotaDb
|
|
538
|
+
const reservation = quotaDb
|
|
539
|
+
.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?")
|
|
540
|
+
.get(token);
|
|
494
541
|
if (!reservation) {
|
|
495
542
|
quotaDb.exec("COMMIT");
|
|
496
543
|
return;
|
|
497
544
|
}
|
|
498
545
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
499
546
|
const used = committedR2Bytes();
|
|
500
|
-
const reserved =
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
547
|
+
const reserved =
|
|
548
|
+
quotaDb
|
|
549
|
+
.query<{ bytes: number }, [string]>(
|
|
550
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
551
|
+
)
|
|
552
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
553
|
+
quotaDb
|
|
554
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
555
|
+
.run(used, reserved, quotaOwner);
|
|
504
556
|
quotaDb.exec("COMMIT");
|
|
505
557
|
} catch (error) {
|
|
506
558
|
quotaDb.exec("ROLLBACK");
|
|
@@ -508,7 +560,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
508
560
|
}
|
|
509
561
|
};
|
|
510
562
|
|
|
511
|
-
const recordR2TransferMetric = (
|
|
563
|
+
const recordR2TransferMetric = (
|
|
564
|
+
field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size",
|
|
565
|
+
): void => {
|
|
512
566
|
if (!quotaDb || !quotaOwner) return;
|
|
513
567
|
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
514
568
|
};
|
|
@@ -526,19 +580,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
526
580
|
};
|
|
527
581
|
|
|
528
582
|
type KvStmts = {
|
|
529
|
-
get: Statement<
|
|
583
|
+
get: Statement<
|
|
584
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
585
|
+
[string, string]
|
|
586
|
+
>;
|
|
530
587
|
put: Statement<unknown, [string, string, string, number | null, number, string | null]>;
|
|
531
588
|
del: Statement<unknown, [string, string]>;
|
|
532
|
-
list: Statement<
|
|
589
|
+
list: Statement<
|
|
590
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
591
|
+
[string, string, string, number, number]
|
|
592
|
+
>;
|
|
533
593
|
};
|
|
534
594
|
const kvStmtCache = new Map<Database, KvStmts>();
|
|
535
595
|
const kvStmts = (store: Database): KvStmts => {
|
|
536
596
|
let stmts = kvStmtCache.get(store);
|
|
537
597
|
if (!stmts) {
|
|
538
598
|
stmts = {
|
|
539
|
-
get: store.query<
|
|
540
|
-
|
|
541
|
-
|
|
599
|
+
get: store.query<
|
|
600
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
601
|
+
[string, string]
|
|
602
|
+
>("SELECT value, expires_at, binary, metadata FROM kv WHERE ns = ? AND key = ?"),
|
|
542
603
|
put: store.query(
|
|
543
604
|
"INSERT INTO kv (ns, key, value, expires_at, binary, metadata) VALUES (?1, ?2, ?3, ?4, ?5, ?6) " +
|
|
544
605
|
"ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5, metadata = ?6",
|
|
@@ -548,7 +609,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
548
609
|
// it is a separate concern (ponytail: rows accumulate until touched by a
|
|
549
610
|
// get/list/put on the same key/prefix, add a cron sweep if that matters).
|
|
550
611
|
// The prefix matches exactly: LIKE ignored case and read "_" and "%" as wildcards.
|
|
551
|
-
list: store.query<
|
|
612
|
+
list: store.query<
|
|
613
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
614
|
+
[string, string, string, number, number]
|
|
615
|
+
>(
|
|
552
616
|
"SELECT key, expires_at, metadata FROM kv WHERE ns = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 " +
|
|
553
617
|
"AND (expires_at IS NULL OR expires_at > ?4) ORDER BY key LIMIT ?5",
|
|
554
618
|
),
|
|
@@ -564,7 +628,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
564
628
|
* seconds) -> ms epoch, or `null` for no expiry. */
|
|
565
629
|
const kvExpiresAt = (msg: Frame): number | null => {
|
|
566
630
|
if (isNumber(msg.expirationTtl)) {
|
|
567
|
-
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
631
|
+
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
632
|
+
throw new Error(`expirationTtl must be at least ${MIN_KV_TTL_SECONDS} seconds`);
|
|
568
633
|
return Date.now() + msg.expirationTtl * 1000;
|
|
569
634
|
}
|
|
570
635
|
if (isNumber(msg.expiration)) return msg.expiration * 1000;
|
|
@@ -759,7 +824,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
759
824
|
});
|
|
760
825
|
};
|
|
761
826
|
|
|
762
|
-
const validateCompletedParts = (
|
|
827
|
+
const validateCompletedParts = (
|
|
828
|
+
stored: R2PartRow[],
|
|
829
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
830
|
+
): void => {
|
|
763
831
|
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
764
832
|
for (let index = 0; index < stored.length; index++) {
|
|
765
833
|
const actual = stored[index];
|
|
@@ -860,7 +928,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
860
928
|
const path = join(directory, name);
|
|
861
929
|
try {
|
|
862
930
|
const tokenValue = name.slice(0, 24);
|
|
863
|
-
const ticket = store
|
|
931
|
+
const ticket = store
|
|
932
|
+
.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?")
|
|
933
|
+
.get(tokenValue);
|
|
864
934
|
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
865
935
|
} catch (error) {
|
|
866
936
|
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
@@ -882,7 +952,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
882
952
|
}
|
|
883
953
|
const start = Number(match[1]);
|
|
884
954
|
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
885
|
-
if (
|
|
955
|
+
if (
|
|
956
|
+
!Number.isSafeInteger(start) ||
|
|
957
|
+
!Number.isSafeInteger(requestedEnd) ||
|
|
958
|
+
start < 0 ||
|
|
959
|
+
start >= size ||
|
|
960
|
+
requestedEnd < start
|
|
961
|
+
)
|
|
962
|
+
return null;
|
|
886
963
|
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
887
964
|
};
|
|
888
965
|
|
|
@@ -909,11 +986,16 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
909
986
|
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
910
987
|
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
911
988
|
}
|
|
912
|
-
if (
|
|
989
|
+
if (
|
|
990
|
+
(ticket.method === "upload" && method !== "PUT") ||
|
|
991
|
+
(ticket.method === "download" && method !== "GET" && method !== "HEAD")
|
|
992
|
+
) {
|
|
913
993
|
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
914
994
|
}
|
|
915
995
|
if (ticket.method === "download") {
|
|
916
|
-
const row = store
|
|
996
|
+
const row = store
|
|
997
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
998
|
+
.get(bucket, ticket.key);
|
|
917
999
|
if (!row) return new Response("not found", { status: 404 });
|
|
918
1000
|
const etag = `"${row.etag}"`;
|
|
919
1001
|
const noneMatch = request.headers.get("if-none-match");
|
|
@@ -945,7 +1027,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
945
1027
|
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
946
1028
|
return new Response("invalid content length", { status: 400 });
|
|
947
1029
|
}
|
|
948
|
-
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1030
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1031
|
+
return new Response("upload too large", { status: 413 });
|
|
949
1032
|
const directory = r2BlobDir(store);
|
|
950
1033
|
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
951
1034
|
if (activeUploads >= maxActiveUploads) {
|
|
@@ -998,7 +1081,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
998
1081
|
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
999
1082
|
renameSync(temporary, destination);
|
|
1000
1083
|
const previous = store
|
|
1001
|
-
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1084
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1085
|
+
"SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?",
|
|
1086
|
+
)
|
|
1002
1087
|
.get(bucket, ticket.key);
|
|
1003
1088
|
const row: R2Row = {
|
|
1004
1089
|
key: ticket.key,
|
|
@@ -1057,7 +1142,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1057
1142
|
try {
|
|
1058
1143
|
for (const part of parts) {
|
|
1059
1144
|
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1060
|
-
if (!bytes || bytes.byteLength !== part.size)
|
|
1145
|
+
if (!bytes || bytes.byteLength !== part.size)
|
|
1146
|
+
throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1061
1147
|
appendFileSync(temporary, bytes);
|
|
1062
1148
|
hash.update(bytes);
|
|
1063
1149
|
size += bytes.byteLength;
|
|
@@ -1103,7 +1189,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1103
1189
|
// group counts as one entry). Same rules as the embedded transport's __sbR2ListPage.
|
|
1104
1190
|
// ponytail: with a delimiter every key after the cursor is read; an index on the
|
|
1105
1191
|
// delimited prefix if buckets grow past ~100k keys.
|
|
1106
|
-
const r2ListPage = (
|
|
1192
|
+
const r2ListPage = (
|
|
1193
|
+
store: Database,
|
|
1194
|
+
bucket: string,
|
|
1195
|
+
prefix: string,
|
|
1196
|
+
cursor: string,
|
|
1197
|
+
limit: number,
|
|
1198
|
+
delimiter: string,
|
|
1199
|
+
) => {
|
|
1107
1200
|
const rows = store
|
|
1108
1201
|
.query<R2Row, [string, string, string]>(
|
|
1109
1202
|
"SELECT * FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 ORDER BY key" +
|
|
@@ -1228,7 +1321,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1228
1321
|
}
|
|
1229
1322
|
}
|
|
1230
1323
|
|
|
1231
|
-
async function proxyFetch(
|
|
1324
|
+
async function proxyFetch(
|
|
1325
|
+
msg: Frame,
|
|
1326
|
+
signal?: AbortSignal,
|
|
1327
|
+
emit?: FrameEmitter,
|
|
1328
|
+
body?: ReadableStream<Uint8Array>,
|
|
1329
|
+
): Promise<Upstream> {
|
|
1232
1330
|
let url: URL;
|
|
1233
1331
|
try {
|
|
1234
1332
|
url = new URL(str(msg.url));
|
|
@@ -1247,7 +1345,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1247
1345
|
const init: RequestInit = {
|
|
1248
1346
|
method,
|
|
1249
1347
|
headers,
|
|
1250
|
-
body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
|
|
1348
|
+
body: method === "GET" || method === "HEAD" ? undefined : (body ?? upstreamBody(msg)),
|
|
1251
1349
|
redirect: "manual",
|
|
1252
1350
|
signal,
|
|
1253
1351
|
};
|
|
@@ -1268,7 +1366,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1268
1366
|
* minus the trip through the network (and therefore minus TLS and the
|
|
1269
1367
|
* default-deny egress rules, which is why this is not `fetch`).
|
|
1270
1368
|
*/
|
|
1271
|
-
async function serviceFetch(
|
|
1369
|
+
async function serviceFetch(
|
|
1370
|
+
msg: Frame,
|
|
1371
|
+
signal?: AbortSignal,
|
|
1372
|
+
emit?: FrameEmitter,
|
|
1373
|
+
body?: ReadableStream<Uint8Array>,
|
|
1374
|
+
): Promise<Upstream> {
|
|
1272
1375
|
const binding = str(msg.binding);
|
|
1273
1376
|
const declared = bindings.services.find((entry) => entry.binding === binding);
|
|
1274
1377
|
if (!declared) throw new Error(`service not bound: ${binding}`);
|
|
@@ -1302,7 +1405,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1302
1405
|
const res = await doFetch(target, {
|
|
1303
1406
|
method,
|
|
1304
1407
|
headers,
|
|
1305
|
-
body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
|
|
1408
|
+
body: method === "GET" || method === "HEAD" ? undefined : (body ?? upstreamBody(msg)),
|
|
1306
1409
|
redirect: "manual",
|
|
1307
1410
|
signal,
|
|
1308
1411
|
});
|
|
@@ -1315,7 +1418,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1315
1418
|
async function dispatch(msg: Frame): Promise<Frame> {
|
|
1316
1419
|
switch (msg.op) {
|
|
1317
1420
|
case "ping":
|
|
1318
|
-
|
|
1421
|
+
// bodyStream: this broker takes streamed request bodies (FRAME_V1_MORE frames after a head).
|
|
1422
|
+
return { ok: true, op: "pong", echo: msg.msg ?? null, pid: process.pid, bodyStream: true };
|
|
1319
1423
|
case "kv.get": {
|
|
1320
1424
|
const { store, part } = storeFor("kv", requireKv(msg.ns));
|
|
1321
1425
|
const stmts = kvStmts(store);
|
|
@@ -1357,7 +1461,13 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1357
1461
|
// last key), with expirations and metadata. Without one: up to 1,000 keys, as before.
|
|
1358
1462
|
const paged = isNumber(msg.limit);
|
|
1359
1463
|
const limit = paged ? Math.max(0, Math.floor(Number(msg.limit))) : 1000;
|
|
1360
|
-
const rows = kvStmts(store).list.all(
|
|
1464
|
+
const rows = kvStmts(store).list.all(
|
|
1465
|
+
part,
|
|
1466
|
+
str(msg.prefix),
|
|
1467
|
+
paged ? str(msg.cursor) : "",
|
|
1468
|
+
Date.now(),
|
|
1469
|
+
paged ? limit + 1 : limit,
|
|
1470
|
+
);
|
|
1361
1471
|
const page = rows.slice(0, limit);
|
|
1362
1472
|
const complete = !paged || rows.length <= limit;
|
|
1363
1473
|
return {
|
|
@@ -1376,17 +1486,22 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1376
1486
|
const name = str(msg.name) || "default";
|
|
1377
1487
|
const key = str(msg.key);
|
|
1378
1488
|
const row = db
|
|
1379
|
-
.query<
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
>("SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?")
|
|
1489
|
+
.query<{ status: number; headers_json: string; body: string; expires_at: number | null }, [string, string]>(
|
|
1490
|
+
"SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?",
|
|
1491
|
+
)
|
|
1383
1492
|
.get(name, key);
|
|
1384
1493
|
if (!row) return { ok: true, found: false };
|
|
1385
1494
|
if (row.expires_at != null && row.expires_at <= Date.now()) {
|
|
1386
1495
|
db.query("DELETE FROM cache WHERE name = ? AND cache_key = ?").run(name, key);
|
|
1387
1496
|
return { ok: true, found: false };
|
|
1388
1497
|
}
|
|
1389
|
-
return {
|
|
1498
|
+
return {
|
|
1499
|
+
ok: true,
|
|
1500
|
+
found: true,
|
|
1501
|
+
status: row.status,
|
|
1502
|
+
headers: jsonObject(parseJsonValue(row.headers_json)) ?? {},
|
|
1503
|
+
body: row.body,
|
|
1504
|
+
};
|
|
1390
1505
|
}
|
|
1391
1506
|
case "cache.put": {
|
|
1392
1507
|
const name = str(msg.name) || "default";
|
|
@@ -1500,9 +1615,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1500
1615
|
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1501
1616
|
const expires = Date.now() + expiresIn * 1000;
|
|
1502
1617
|
const maxBytes =
|
|
1503
|
-
method === "upload"
|
|
1504
|
-
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1505
|
-
: 0;
|
|
1618
|
+
method === "upload" ? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit()) : 0;
|
|
1506
1619
|
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1507
1620
|
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1508
1621
|
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
@@ -1562,11 +1675,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1562
1675
|
case "r2.head": {
|
|
1563
1676
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1564
1677
|
const key = str(msg.key);
|
|
1565
|
-
const row = store
|
|
1678
|
+
const row = store
|
|
1679
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1680
|
+
.get(bucket, key);
|
|
1566
1681
|
if (!row) return { ok: true, found: false };
|
|
1567
|
-
const body =
|
|
1568
|
-
|
|
1569
|
-
|
|
1682
|
+
const body =
|
|
1683
|
+
msg.op === "r2.get"
|
|
1684
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1685
|
+
: undefined;
|
|
1570
1686
|
return { ok: true, found: true, object: r2Row(row), body };
|
|
1571
1687
|
}
|
|
1572
1688
|
case "r2.delete": {
|
|
@@ -1582,7 +1698,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1582
1698
|
case "r2.md5": {
|
|
1583
1699
|
// Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
|
|
1584
1700
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1585
|
-
store
|
|
1701
|
+
store
|
|
1702
|
+
.query("UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4")
|
|
1703
|
+
.run(str(msg.md5), bucket, str(msg.key), str(msg.etag));
|
|
1586
1704
|
return { ok: true };
|
|
1587
1705
|
}
|
|
1588
1706
|
case "r2.list": {
|
|
@@ -1670,10 +1788,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1670
1788
|
return row ? { ok: true, found: true, value: row.value } : { ok: true, found: false };
|
|
1671
1789
|
}
|
|
1672
1790
|
case "do.storage.put":
|
|
1673
|
-
doDb
|
|
1674
|
-
|
|
1675
|
-
"
|
|
1676
|
-
|
|
1791
|
+
doDb
|
|
1792
|
+
.query(
|
|
1793
|
+
"INSERT INTO do_storage (cls, id, key, value) VALUES (?1,?2,?3,?4) " +
|
|
1794
|
+
"ON CONFLICT (cls, id, key) DO UPDATE SET value = ?4",
|
|
1795
|
+
)
|
|
1796
|
+
.run(requireDoClass(msg.cls), str(msg.id), str(msg.key), str(msg.value));
|
|
1677
1797
|
return { ok: true };
|
|
1678
1798
|
case "do.storage.delete": {
|
|
1679
1799
|
const r = doDb
|
|
@@ -1687,11 +1807,20 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1687
1807
|
case "do.storage.list": {
|
|
1688
1808
|
const cls = requireDoClass(msg.cls);
|
|
1689
1809
|
const limit = Math.min(Math.max(Number(msg.limit) || 1000, 1), 10000);
|
|
1810
|
+
// start, startAfter, end and reverse come from the runtime's 2026-10-28 storage.
|
|
1811
|
+
// The prefix match is exact: LIKE also matched _ and % as wildcards, case-insensitively.
|
|
1812
|
+
const bound = (v: JsonValue | undefined): string | null => (v == null ? null : str(v));
|
|
1690
1813
|
const rows = doDb
|
|
1691
|
-
.query<
|
|
1692
|
-
|
|
1814
|
+
.query<
|
|
1815
|
+
{ key: string; value: string },
|
|
1816
|
+
[string, string, string, number, string | null, string | null, string | null]
|
|
1817
|
+
>(
|
|
1818
|
+
"SELECT key, value FROM do_storage WHERE cls = ?1 AND id = ?2 AND substr(key, 1, length(?3)) = ?3" +
|
|
1819
|
+
" AND (?5 IS NULL OR key >= ?5) AND (?6 IS NULL OR key > ?6) AND (?7 IS NULL OR key < ?7)" +
|
|
1820
|
+
(msg.reverse === true ? " ORDER BY key DESC" : " ORDER BY key") +
|
|
1821
|
+
" LIMIT ?4",
|
|
1693
1822
|
)
|
|
1694
|
-
.all(cls, str(msg.id), str(msg.prefix), limit);
|
|
1823
|
+
.all(cls, str(msg.id), str(msg.prefix), limit, bound(msg.start), bound(msg.startAfter), bound(msg.end));
|
|
1695
1824
|
return { ok: true, entries: rows.map((r) => [r.key, r.value]) };
|
|
1696
1825
|
}
|
|
1697
1826
|
|
|
@@ -1699,17 +1828,23 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1699
1828
|
// for that object, and attempts resets because this is a fresh schedule.
|
|
1700
1829
|
case "do.alarm.set": {
|
|
1701
1830
|
const cls = requireDoClass(msg.cls);
|
|
1702
|
-
doDb
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1831
|
+
doDb
|
|
1832
|
+
.transaction(() => {
|
|
1833
|
+
doDb
|
|
1834
|
+
.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=? AND expires<=?")
|
|
1835
|
+
.run(cls, str(msg.id), Date.now());
|
|
1836
|
+
doDb
|
|
1837
|
+
.query(
|
|
1838
|
+
"INSERT INTO do_alarm (cls,id,at,attempts) VALUES (?1,?2,?3,0) ON CONFLICT (cls,id) DO UPDATE SET at=?3,attempts=0",
|
|
1839
|
+
)
|
|
1840
|
+
.run(cls, str(msg.id), Math.trunc(Number(msg.at) || 0));
|
|
1841
|
+
})
|
|
1842
|
+
.immediate();
|
|
1843
|
+
return { ok: true };
|
|
1707
1844
|
}
|
|
1708
1845
|
case "do.alarm.get": {
|
|
1709
1846
|
const row = doDb
|
|
1710
|
-
.query<{ at: number }, [string, string]>(
|
|
1711
|
-
"SELECT at FROM do_alarm WHERE cls = ? AND id = ?",
|
|
1712
|
-
)
|
|
1847
|
+
.query<{ at: number }, [string, string]>("SELECT at FROM do_alarm WHERE cls = ? AND id = ?")
|
|
1713
1848
|
.get(requireDoClass(msg.cls), str(msg.id));
|
|
1714
1849
|
const retry = row
|
|
1715
1850
|
? null
|
|
@@ -1722,12 +1857,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1722
1857
|
}
|
|
1723
1858
|
case "do.alarm.delete": {
|
|
1724
1859
|
const cls = requireDoClass(msg.cls);
|
|
1725
|
-
const deleted = doDb
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1860
|
+
const deleted = doDb
|
|
1861
|
+
.transaction(() => {
|
|
1862
|
+
const pending = doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1863
|
+
const delivery = doDb.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1864
|
+
return pending.changes > 0 || delivery.changes > 0;
|
|
1865
|
+
})
|
|
1866
|
+
.immediate();
|
|
1867
|
+
return { ok: true, deleted };
|
|
1731
1868
|
}
|
|
1732
1869
|
|
|
1733
1870
|
case "assets.get": {
|
|
@@ -1785,6 +1922,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1785
1922
|
binary: Uint8Array,
|
|
1786
1923
|
signal?: AbortSignal,
|
|
1787
1924
|
emit?: FrameEmitter,
|
|
1925
|
+
body?: ReadableStream<Uint8Array>,
|
|
1788
1926
|
): Promise<{ reply: Frame; bytes?: Uint8Array }> {
|
|
1789
1927
|
if (msg.op === "r2.put") {
|
|
1790
1928
|
// The body arrived as bytes; write it straight to its own file rather
|
|
@@ -1826,7 +1964,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1826
1964
|
if (msg.op === "r2.get") {
|
|
1827
1965
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1828
1966
|
const key = str(msg.key);
|
|
1829
|
-
const row = store
|
|
1967
|
+
const row = store
|
|
1968
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1969
|
+
.get(bucket, key);
|
|
1830
1970
|
if (!row) return { reply: { ok: true, found: false } };
|
|
1831
1971
|
return {
|
|
1832
1972
|
reply: { ok: true, found: true, object: r2Row(row) },
|
|
@@ -1835,8 +1975,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1835
1975
|
}
|
|
1836
1976
|
|
|
1837
1977
|
// #232 — a v1 fetch gets the upstream body as raw bytes, not decoded text.
|
|
1838
|
-
if (msg.op === "fetch") return proxyFetch(msg, signal, emit);
|
|
1839
|
-
if (msg.op === "service.fetch") return serviceFetch(msg, signal, emit);
|
|
1978
|
+
if (msg.op === "fetch") return proxyFetch(msg, signal, emit, body);
|
|
1979
|
+
if (msg.op === "service.fetch") return serviceFetch(msg, signal, emit, body);
|
|
1840
1980
|
|
|
1841
1981
|
if (msg.op === "assets.get") {
|
|
1842
1982
|
if (!bindings.assets) throw new Error("assets not bound");
|
|
@@ -1857,7 +1997,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1857
1997
|
} else if (!hit && assetManifest?.notFound === "404-page") {
|
|
1858
1998
|
hit = readAsset("/404.html");
|
|
1859
1999
|
}
|
|
1860
|
-
if (!hit)
|
|
2000
|
+
if (!hit)
|
|
2001
|
+
return { reply: { ok: true, found: false, status: 404, rules }, bytes: new TextEncoder().encode("Not Found") };
|
|
1861
2002
|
return {
|
|
1862
2003
|
reply: { ok: true, found, status, type: hit.type, hash: hit.hash, rules },
|
|
1863
2004
|
bytes: hit.body,
|
|
@@ -1868,7 +2009,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1868
2009
|
}
|
|
1869
2010
|
|
|
1870
2011
|
/** One wire frame in, one out. See `handleFrame` on the Broker type. */
|
|
1871
|
-
async function handleFrame(
|
|
2012
|
+
async function handleFrame(
|
|
2013
|
+
payload: Buffer,
|
|
2014
|
+
signal?: AbortSignal,
|
|
2015
|
+
emit?: FrameEmitter,
|
|
2016
|
+
body?: ReadableStream<Uint8Array>,
|
|
2017
|
+
): Promise<Buffer> {
|
|
1872
2018
|
if (payload.length === 0 || payload[0] !== FRAME_V1) {
|
|
1873
2019
|
return encodeFrame(await handlePayload(payload.toString("utf8")));
|
|
1874
2020
|
}
|
|
@@ -1884,7 +2030,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1884
2030
|
const seen = replayed.get(replayKey);
|
|
1885
2031
|
if (seen) return frameOf(encodeV1(seen));
|
|
1886
2032
|
}
|
|
1887
|
-
const { reply, bytes } = await dispatchBinary(msg, binary, signal, emit);
|
|
2033
|
+
const { reply, bytes } = await dispatchBinary(msg, binary, signal, emit, body);
|
|
1888
2034
|
if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
|
|
1889
2035
|
return frameOf(encodeV1(reply, bytes));
|
|
1890
2036
|
} catch (e) {
|
|
@@ -1979,14 +2125,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1979
2125
|
|
|
1980
2126
|
const timeoutMs = opts.deliveryTimeoutMs ?? 25_000;
|
|
1981
2127
|
const leaseMs = opts.deliveryLeaseMs ?? 30_000;
|
|
1982
|
-
if (!(timeoutMs > 0 && leaseMs > timeoutMs))
|
|
1983
|
-
throw new Error("delivery lease must exceed its timeout");
|
|
2128
|
+
if (!(timeoutMs > 0 && leaseMs > timeoutMs)) throw new Error("delivery lease must exceed its timeout");
|
|
1984
2129
|
let closed = false;
|
|
1985
2130
|
const deliveries = new Set<AbortController>();
|
|
1986
|
-
async function deliverTrigger(
|
|
1987
|
-
kind: "scheduled" | "queue" | "alarm",
|
|
1988
|
-
body: JsonObject,
|
|
1989
|
-
): Promise<Response | null> {
|
|
2131
|
+
async function deliverTrigger(kind: "scheduled" | "queue" | "alarm", body: JsonObject): Promise<Response | null> {
|
|
1990
2132
|
if (!opts.sproutUrl || !dispatchEnabled() || closed) return null;
|
|
1991
2133
|
const abort = new AbortController();
|
|
1992
2134
|
deliveries.add(abort);
|
|
@@ -2003,21 +2145,30 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2003
2145
|
},
|
|
2004
2146
|
body: JSON.stringify(body),
|
|
2005
2147
|
}).then(async (res) => {
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2148
|
+
if (kind !== "queue") {
|
|
2149
|
+
await res.body?.cancel();
|
|
2150
|
+
return new Response(null, { status: res.status });
|
|
2151
|
+
}
|
|
2152
|
+
const chunks: Uint8Array[] = [];
|
|
2153
|
+
let size = 0;
|
|
2154
|
+
const reader = res.body?.getReader();
|
|
2155
|
+
try {
|
|
2156
|
+
if (reader)
|
|
2157
|
+
for (;;) {
|
|
2012
2158
|
const part = await reader.read();
|
|
2013
2159
|
if (part.done) break;
|
|
2014
2160
|
size += part.value.byteLength;
|
|
2015
|
-
if (size > 65_536) {
|
|
2161
|
+
if (size > 65_536) {
|
|
2162
|
+
await reader.cancel();
|
|
2163
|
+
throw new Error("delivery acknowledgement exceeds 64 KiB");
|
|
2164
|
+
}
|
|
2016
2165
|
chunks.push(part.value);
|
|
2017
2166
|
}
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
}
|
|
2167
|
+
} finally {
|
|
2168
|
+
reader?.releaseLock();
|
|
2169
|
+
}
|
|
2170
|
+
return new Response(size ? Buffer.concat(chunks) : null, { status: res.status, headers: res.headers });
|
|
2171
|
+
}),
|
|
2021
2172
|
new Promise<never>((_, reject) =>
|
|
2022
2173
|
abort.signal.addEventListener("abort", () => reject(new Error("delivery interrupted")), {
|
|
2023
2174
|
once: true,
|
|
@@ -2094,9 +2245,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2094
2245
|
for (const id of ack)
|
|
2095
2246
|
if (!retrySet.has(id))
|
|
2096
2247
|
store
|
|
2097
|
-
.query(
|
|
2098
|
-
"DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?",
|
|
2099
|
-
)
|
|
2248
|
+
.query("DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?")
|
|
2100
2249
|
.run(part, id, claim, Date.now());
|
|
2101
2250
|
for (const id of retry)
|
|
2102
2251
|
store
|
|
@@ -2116,7 +2265,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2116
2265
|
const now = Date.now();
|
|
2117
2266
|
const claimed = doDb
|
|
2118
2267
|
.transaction(() => {
|
|
2119
|
-
doDb
|
|
2268
|
+
doDb
|
|
2269
|
+
.query(
|
|
2270
|
+
"DELETE FROM do_alarm_delivery WHERE expires<=? AND attempts>=5 AND EXISTS (SELECT 1 FROM do_alarm a WHERE a.cls=do_alarm_delivery.cls AND a.id=do_alarm_delivery.id)",
|
|
2271
|
+
)
|
|
2272
|
+
.run(now);
|
|
2120
2273
|
const due = doDb
|
|
2121
2274
|
.query<{ cls: string; id: string; at: number; attempts: number }, [number]>(
|
|
2122
2275
|
"SELECT cls,id,at,attempts FROM do_alarm WHERE at <= ? AND NOT EXISTS (SELECT 1 FROM do_alarm_delivery d WHERE d.cls=do_alarm.cls AND d.id=do_alarm.id) ORDER BY at LIMIT 10",
|
|
@@ -2124,9 +2277,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2124
2277
|
.all(now)
|
|
2125
2278
|
.filter((r) => bindings.do.some((b) => b.className === r.cls));
|
|
2126
2279
|
for (const r of due) {
|
|
2127
|
-
doDb
|
|
2128
|
-
.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)")
|
|
2129
|
-
.run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2280
|
+
doDb.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)").run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2130
2281
|
doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=? AND at=?").run(r.cls, r.id, r.at);
|
|
2131
2282
|
}
|
|
2132
2283
|
const expired = doDb
|
|
@@ -2138,9 +2289,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2138
2289
|
return expired.map((r) => {
|
|
2139
2290
|
const claim = randomBytes(24).toString("hex");
|
|
2140
2291
|
doDb
|
|
2141
|
-
.query(
|
|
2142
|
-
"UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?",
|
|
2143
|
-
)
|
|
2292
|
+
.query("UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?")
|
|
2144
2293
|
.run(claim, now + leaseMs, r.cls, r.id);
|
|
2145
2294
|
return { ...r, claim };
|
|
2146
2295
|
});
|
|
@@ -2155,9 +2304,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2155
2304
|
.run(r.cls, r.id, r.claim, Date.now());
|
|
2156
2305
|
else
|
|
2157
2306
|
doDb
|
|
2158
|
-
.query(
|
|
2159
|
-
"UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?",
|
|
2160
|
-
)
|
|
2307
|
+
.query("UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?")
|
|
2161
2308
|
.run(Date.now() + 5_000, r.cls, r.id, r.claim, Date.now());
|
|
2162
2309
|
});
|
|
2163
2310
|
}
|
|
@@ -2175,8 +2322,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2175
2322
|
if (stamp === lastTick) return; // once per minute
|
|
2176
2323
|
lastTick = stamp;
|
|
2177
2324
|
for (const expr of bindings.crons) {
|
|
2178
|
-
if (cronMatches(expr, now))
|
|
2179
|
-
void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2325
|
+
if (cronMatches(expr, now)) void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2180
2326
|
}
|
|
2181
2327
|
}, 15_000),
|
|
2182
2328
|
);
|
|
@@ -2238,6 +2384,85 @@ export function encodeFrame(obj: Frame): Buffer {
|
|
|
2238
2384
|
/** The bound TCP listener: the port actually assigned, and its shutdown handle. */
|
|
2239
2385
|
export type BrokerServer = { port: number; stop(): void };
|
|
2240
2386
|
|
|
2387
|
+
/** Whether a payload is the head of a fetch whose request body follows in FRAME_V1_MORE frames. */
|
|
2388
|
+
function bodyStreamHead(payload: Buffer): boolean {
|
|
2389
|
+
if (payload.length < 5 || payload[0] !== FRAME_V1) return false;
|
|
2390
|
+
const jsonLen = payload.readUInt32LE(1);
|
|
2391
|
+
if (5 + jsonLen > payload.length) return false;
|
|
2392
|
+
const head = payload.subarray(5, 5 + jsonLen).toString("utf8");
|
|
2393
|
+
// Cheap first: most frames are not this.
|
|
2394
|
+
if (!head.includes('"bodyStream":true')) return false;
|
|
2395
|
+
const msg = jsonObject(parseJsonValue(head));
|
|
2396
|
+
return msg?.bodyStream === true;
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
/** The request body of a streamed-body request, fed by the frames after its head. */
|
|
2400
|
+
type BodySink = {
|
|
2401
|
+
stream: ReadableStream<Uint8Array>;
|
|
2402
|
+
/** A FRAME_V1_MORE payload: `{ chunk: true }` plus bytes, `{ end: true }`, or `{ error }`. */
|
|
2403
|
+
frame(payload: Buffer): void;
|
|
2404
|
+
fail(reason: string): void;
|
|
2405
|
+
};
|
|
2406
|
+
/** Bytes a streamed request body may hold unread before the sprout's connection is paused. */
|
|
2407
|
+
const BODY_SINK_HIGH = 4 * 1024 * 1024;
|
|
2408
|
+
function bodySink(socket: { pause(): void; resume(): void }): BodySink {
|
|
2409
|
+
let controller: ReadableStreamDefaultController<Uint8Array> | null = null;
|
|
2410
|
+
let finished = false;
|
|
2411
|
+
let paused = false;
|
|
2412
|
+
const resume = () => {
|
|
2413
|
+
if (paused) {
|
|
2414
|
+
paused = false;
|
|
2415
|
+
socket.resume();
|
|
2416
|
+
}
|
|
2417
|
+
};
|
|
2418
|
+
const finish = () => {
|
|
2419
|
+
finished = true;
|
|
2420
|
+
resume();
|
|
2421
|
+
};
|
|
2422
|
+
const stream = new ReadableStream<Uint8Array>(
|
|
2423
|
+
{
|
|
2424
|
+
start(c) {
|
|
2425
|
+
controller = c;
|
|
2426
|
+
},
|
|
2427
|
+
pull() {
|
|
2428
|
+
resume();
|
|
2429
|
+
},
|
|
2430
|
+
cancel() {
|
|
2431
|
+
finish();
|
|
2432
|
+
},
|
|
2433
|
+
},
|
|
2434
|
+
new ByteLengthQueuingStrategy({ highWaterMark: BODY_SINK_HIGH }),
|
|
2435
|
+
);
|
|
2436
|
+
return {
|
|
2437
|
+
stream,
|
|
2438
|
+
frame(payload) {
|
|
2439
|
+
if (finished || !controller || payload.length < 5) return;
|
|
2440
|
+
const jsonLen = payload.readUInt32LE(1);
|
|
2441
|
+
const msg = jsonObject(parseJsonValue(payload.subarray(5, 5 + jsonLen).toString("utf8")));
|
|
2442
|
+
if (msg?.chunk === true) {
|
|
2443
|
+
const bytes = payload.subarray(5 + jsonLen);
|
|
2444
|
+
if (bytes.length) controller.enqueue(new Uint8Array(bytes));
|
|
2445
|
+
if ((controller.desiredSize ?? 1) <= 0 && !paused) {
|
|
2446
|
+
paused = true;
|
|
2447
|
+
socket.pause();
|
|
2448
|
+
}
|
|
2449
|
+
return;
|
|
2450
|
+
}
|
|
2451
|
+
finish();
|
|
2452
|
+
if (msg?.end === true) controller.close();
|
|
2453
|
+
else {
|
|
2454
|
+
const error = msg?.error;
|
|
2455
|
+
controller.error(new Error(isString(error) ? error : "the request body failed"));
|
|
2456
|
+
}
|
|
2457
|
+
},
|
|
2458
|
+
fail(reason) {
|
|
2459
|
+
if (finished) return;
|
|
2460
|
+
finish();
|
|
2461
|
+
controller?.error(new Error(reason));
|
|
2462
|
+
},
|
|
2463
|
+
};
|
|
2464
|
+
}
|
|
2465
|
+
|
|
2241
2466
|
/** Start the TCP listener. Returns the bound port. */
|
|
2242
2467
|
export function listen(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
2243
2468
|
// Chunks are held in a list and joined once, when a whole frame has arrived.
|
|
@@ -2254,12 +2479,17 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2254
2479
|
written: (() => void) | null;
|
|
2255
2480
|
/** Aborted when the connection closes, so upstream requests it started stop. */
|
|
2256
2481
|
abort: AbortController;
|
|
2482
|
+
/** The request body a streamed-body request on this connection is reading, if any. */
|
|
2483
|
+
sink: BodySink | null;
|
|
2257
2484
|
};
|
|
2258
2485
|
const flush = (socket: Bun.Socket<SocketState>): void => {
|
|
2259
2486
|
const state = socket.data;
|
|
2260
2487
|
if (state.closed || !state.output) return;
|
|
2261
2488
|
const accepted = socket.write(state.output.subarray(state.offset));
|
|
2262
|
-
if (accepted < 0) {
|
|
2489
|
+
if (accepted < 0) {
|
|
2490
|
+
socket.end();
|
|
2491
|
+
return;
|
|
2492
|
+
}
|
|
2263
2493
|
state.offset += accepted;
|
|
2264
2494
|
if (state.offset === state.output.length) {
|
|
2265
2495
|
state.output = null;
|
|
@@ -2291,6 +2521,7 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2291
2521
|
offset: 0,
|
|
2292
2522
|
written: null,
|
|
2293
2523
|
abort: new AbortController(),
|
|
2524
|
+
sink: null,
|
|
2294
2525
|
};
|
|
2295
2526
|
},
|
|
2296
2527
|
drain: flush,
|
|
@@ -2301,6 +2532,7 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2301
2532
|
state.chunks = [];
|
|
2302
2533
|
state.size = 0;
|
|
2303
2534
|
state.output = null;
|
|
2535
|
+
state.sink?.fail("the sprout closed the connection");
|
|
2304
2536
|
const done = state.written;
|
|
2305
2537
|
state.written = null;
|
|
2306
2538
|
done?.();
|
|
@@ -2311,30 +2543,56 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2311
2543
|
state.size += chunk.length;
|
|
2312
2544
|
if (state.processing) return;
|
|
2313
2545
|
state.processing = true;
|
|
2314
|
-
try {
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2546
|
+
try {
|
|
2547
|
+
for (;;) {
|
|
2548
|
+
if (state.closed) return;
|
|
2549
|
+
if (state.size < 4) return;
|
|
2550
|
+
if (state.chunks.length > 1) {
|
|
2551
|
+
state.chunks = [Buffer.concat(state.chunks, state.size)];
|
|
2552
|
+
}
|
|
2553
|
+
const buf = state.chunks[0];
|
|
2554
|
+
const len = buf.readUInt32LE(0);
|
|
2555
|
+
if (len > MAX_FRAME) {
|
|
2556
|
+
await send(socket, encodeFrame({ ok: false, error: "frame too large" }));
|
|
2557
|
+
socket.end();
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
if (state.size < 4 + len) return;
|
|
2561
|
+
// Kept as bytes: a v1 frame carries binary after its JSON, and
|
|
2562
|
+
// decoding the whole payload as utf8 would corrupt it.
|
|
2563
|
+
const payload = buf.subarray(4, 4 + len);
|
|
2564
|
+
const rest = buf.subarray(4 + len);
|
|
2565
|
+
state.chunks = rest.length ? [Buffer.from(rest)] : [];
|
|
2566
|
+
state.size = rest.length;
|
|
2567
|
+
// A frame of a streamed request body: to its request's stream.
|
|
2568
|
+
if (payload.length > 0 && payload[0] === FRAME_V1_MORE) {
|
|
2569
|
+
state.sink?.frame(payload);
|
|
2570
|
+
continue;
|
|
2571
|
+
}
|
|
2572
|
+
// A request whose body follows in frames: it runs while this loop feeds them.
|
|
2573
|
+
if (bodyStreamHead(payload)) {
|
|
2574
|
+
const sink = bodySink(socket);
|
|
2575
|
+
state.sink = sink;
|
|
2576
|
+
void (async () => {
|
|
2577
|
+
const reply = await broker.handleFrame(
|
|
2578
|
+
payload,
|
|
2579
|
+
state.abort.signal,
|
|
2580
|
+
(frame) => send(socket, frame),
|
|
2581
|
+
sink.stream,
|
|
2582
|
+
);
|
|
2583
|
+
await send(socket, reply);
|
|
2584
|
+
// Body frames the sprout still sends (the upstream answered early) are dropped.
|
|
2585
|
+
sink.fail("the request already ended");
|
|
2586
|
+
})();
|
|
2587
|
+
continue;
|
|
2588
|
+
}
|
|
2589
|
+
// Bun does not buffer unwritten bytes. Serialize replies and retain
|
|
2590
|
+
// the unsent suffix until drain, including binary R2 responses.
|
|
2591
|
+
await send(socket, await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame)));
|
|
2326
2592
|
}
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
const payload = buf.subarray(4, 4 + len);
|
|
2331
|
-
const rest = buf.subarray(4 + len);
|
|
2332
|
-
state.chunks = rest.length ? [Buffer.from(rest)] : [];
|
|
2333
|
-
state.size = rest.length;
|
|
2334
|
-
// Bun does not buffer unwritten bytes. Serialize replies and retain
|
|
2335
|
-
// the unsent suffix until drain, including binary R2 responses.
|
|
2336
|
-
await send(socket, await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame)));
|
|
2337
|
-
} } finally { state.processing = false; }
|
|
2593
|
+
} finally {
|
|
2594
|
+
state.processing = false;
|
|
2595
|
+
}
|
|
2338
2596
|
},
|
|
2339
2597
|
},
|
|
2340
2598
|
});
|
|
@@ -2345,10 +2603,15 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2345
2603
|
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
2346
2604
|
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
2347
2605
|
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
2348
|
-
const server = Bun.serve({
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2606
|
+
const server = Bun.serve({
|
|
2607
|
+
hostname,
|
|
2608
|
+
port,
|
|
2609
|
+
maxRequestBodySize,
|
|
2610
|
+
fetch(request, server) {
|
|
2611
|
+
server.timeout(request, 255);
|
|
2612
|
+
return broker.transfer(request);
|
|
2613
|
+
},
|
|
2614
|
+
});
|
|
2352
2615
|
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
2353
2616
|
}
|
|
2354
2617
|
|
|
@@ -2394,9 +2657,7 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2394
2657
|
const parsed = jsonObject(parseJsonValue(values.services));
|
|
2395
2658
|
if (parsed) {
|
|
2396
2659
|
services = Object.fromEntries(
|
|
2397
|
-
Object.entries(parsed).flatMap(([binding, host]) =>
|
|
2398
|
-
isString(host) ? [[binding, host] as const] : [],
|
|
2399
|
-
),
|
|
2660
|
+
Object.entries(parsed).flatMap(([binding, host]) => (isString(host) ? [[binding, host] as const] : [])),
|
|
2400
2661
|
);
|
|
2401
2662
|
}
|
|
2402
2663
|
} catch {
|
|
@@ -2414,13 +2675,9 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2414
2675
|
resourceDir: values["resource-dir"],
|
|
2415
2676
|
doDb: values["do-db"],
|
|
2416
2677
|
ownerId: values["owner-id"],
|
|
2417
|
-
r2QuotaBytes:
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
? String(values["r2-resource-ids"]).split(",").filter(Boolean)
|
|
2421
|
-
: undefined,
|
|
2422
|
-
r2MinFreeBytes:
|
|
2423
|
-
Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2678
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2679
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2680
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2424
2681
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
2425
2682
|
bindings,
|
|
2426
2683
|
secrets,
|