@sproutboat/wire 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/package.json +2 -2
- package/src/broker.test.ts +370 -97
- package/src/broker.ts +348 -170
- package/src/d1.ts +19 -2
package/src/broker.ts
CHANGED
|
@@ -21,7 +21,18 @@ import { createD1Store } from "./d1";
|
|
|
21
21
|
import type { D1Pool } from "./d1-pool";
|
|
22
22
|
import { Database, type Statement } from "bun:sqlite";
|
|
23
23
|
import { createHash, randomBytes } from "node:crypto";
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
appendFileSync,
|
|
26
|
+
existsSync,
|
|
27
|
+
mkdirSync,
|
|
28
|
+
readFileSync,
|
|
29
|
+
readdirSync,
|
|
30
|
+
renameSync,
|
|
31
|
+
rmSync,
|
|
32
|
+
statfsSync,
|
|
33
|
+
statSync,
|
|
34
|
+
writeFileSync,
|
|
35
|
+
} from "node:fs";
|
|
25
36
|
import { open } from "node:fs/promises";
|
|
26
37
|
import { dirname, join, normalize, resolve } from "node:path";
|
|
27
38
|
import { parseArgs } from "node:util";
|
|
@@ -200,7 +211,12 @@ export type Broker = {
|
|
|
200
211
|
* for it (`bodyStream: true`) sends that stream as its request body: the listener feeds
|
|
201
212
|
* it from the FRAME_V1_MORE frames that follow the head on the same connection.
|
|
202
213
|
*/
|
|
203
|
-
handleFrame(
|
|
214
|
+
handleFrame(
|
|
215
|
+
payload: Buffer,
|
|
216
|
+
signal?: AbortSignal,
|
|
217
|
+
emit?: FrameEmitter,
|
|
218
|
+
body?: ReadableStream<Uint8Array>,
|
|
219
|
+
): Promise<Buffer>;
|
|
204
220
|
/** Serve one reserved direct R2 upload or download request. */
|
|
205
221
|
transfer(request: Request): Promise<Response>;
|
|
206
222
|
close(): void;
|
|
@@ -300,10 +316,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
300
316
|
"expires_at INTEGER, PRIMARY KEY (ns, key))",
|
|
301
317
|
);
|
|
302
318
|
const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
|
|
303
|
-
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
319
|
+
if (!kvColumns.some((column) => column.name === "expires_at"))
|
|
320
|
+
conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
|
|
304
321
|
// #233 — 1 when the value was put as bytes (stored one char per byte), so get()
|
|
305
322
|
// can hand back exact bytes or decoded text depending on the requested type.
|
|
306
|
-
if (!kvColumns.some((column) => column.name === "binary"))
|
|
323
|
+
if (!kvColumns.some((column) => column.name === "binary"))
|
|
324
|
+
conn.exec("ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0");
|
|
307
325
|
// A key's metadata (put(key, value, { metadata })), as JSON text.
|
|
308
326
|
if (!kvColumns.some((column) => column.name === "metadata")) conn.exec("ALTER TABLE kv ADD COLUMN metadata TEXT");
|
|
309
327
|
// #56 — no `body` column: an object's bytes live in their own file under
|
|
@@ -344,7 +362,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
344
362
|
// Every queue poll filters one partition's live rows and takes its earliest
|
|
345
363
|
// visible batch. This additive index also upgrades existing resource files.
|
|
346
364
|
conn.exec("CREATE INDEX IF NOT EXISTS mq_due ON mq (queue, dead, visible_at)");
|
|
347
|
-
if (
|
|
365
|
+
if (
|
|
366
|
+
!conn
|
|
367
|
+
.query<{ name: string }, []>("PRAGMA table_info(mq)")
|
|
368
|
+
.all()
|
|
369
|
+
.some((c) => c.name === "claim_token")
|
|
370
|
+
)
|
|
348
371
|
conn.exec("ALTER TABLE mq ADD COLUMN claim_token TEXT NOT NULL DEFAULT ''");
|
|
349
372
|
return conn;
|
|
350
373
|
};
|
|
@@ -373,7 +396,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
373
396
|
);
|
|
374
397
|
// Alarm polling reads the earliest due objects across classes.
|
|
375
398
|
doDb.exec("CREATE INDEX IF NOT EXISTS do_alarm_due ON do_alarm (at)");
|
|
376
|
-
doDb.exec(
|
|
399
|
+
doDb.exec(
|
|
400
|
+
"CREATE TABLE IF NOT EXISTS do_alarm_delivery (cls TEXT NOT NULL, id TEXT NOT NULL, at INTEGER NOT NULL, attempts INTEGER NOT NULL, token TEXT NOT NULL, expires INTEGER NOT NULL, PRIMARY KEY(cls,id))",
|
|
401
|
+
);
|
|
377
402
|
// #69 — fixed-window rate-limiter counters, per binding name + key.
|
|
378
403
|
db.exec(
|
|
379
404
|
"CREATE TABLE IF NOT EXISTS ratelimit (name TEXT NOT NULL, key TEXT NOT NULL, window_start INTEGER NOT NULL, " +
|
|
@@ -407,9 +432,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
407
432
|
// One ledger for every account, shared by the brokers of all its deployments.
|
|
408
433
|
// Ticket reservations are recorded before an upload begins, so four accepted
|
|
409
434
|
// requests cannot each independently decide that the same remaining bytes fit.
|
|
410
|
-
const quotaDb =
|
|
411
|
-
|
|
412
|
-
|
|
435
|
+
const quotaDb =
|
|
436
|
+
quotaOwner && quotaDirectory && quotaBytes
|
|
437
|
+
? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
|
|
438
|
+
: null;
|
|
413
439
|
if (quotaDb) {
|
|
414
440
|
const owner = quotaOwner!;
|
|
415
441
|
quotaDb.exec("PRAGMA journal_mode = WAL");
|
|
@@ -422,7 +448,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
422
448
|
quotaDb.exec(
|
|
423
449
|
"CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
|
|
424
450
|
);
|
|
425
|
-
const existing = quotaDb
|
|
451
|
+
const existing = quotaDb
|
|
452
|
+
.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?")
|
|
453
|
+
.get(owner);
|
|
426
454
|
if (!existing) {
|
|
427
455
|
quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
|
|
428
456
|
}
|
|
@@ -436,9 +464,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
436
464
|
// Open through resourceDb so older resource files gain the multipart
|
|
437
465
|
// tables before the aggregate is queried.
|
|
438
466
|
const store = resourceDb(id);
|
|
439
|
-
used +=
|
|
440
|
-
|
|
441
|
-
|
|
467
|
+
used +=
|
|
468
|
+
store
|
|
469
|
+
.query<{ bytes: number }, []>(
|
|
470
|
+
"SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
|
|
471
|
+
)
|
|
472
|
+
.get()?.bytes ?? 0;
|
|
442
473
|
}
|
|
443
474
|
return used;
|
|
444
475
|
};
|
|
@@ -453,9 +484,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
453
484
|
const releaseExpiredR2Reservations = (now = Date.now()): void => {
|
|
454
485
|
if (!quotaDb || !quotaOwner) return;
|
|
455
486
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
|
|
456
|
-
const reserved =
|
|
457
|
-
|
|
458
|
-
|
|
487
|
+
const reserved =
|
|
488
|
+
quotaDb
|
|
489
|
+
.query<{ bytes: number }, [string]>(
|
|
490
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
491
|
+
)
|
|
492
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
459
493
|
quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
|
|
460
494
|
};
|
|
461
495
|
|
|
@@ -465,10 +499,15 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
465
499
|
try {
|
|
466
500
|
releaseExpiredR2Reservations();
|
|
467
501
|
const used = committedR2Bytes();
|
|
468
|
-
const reserved =
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
502
|
+
const reserved =
|
|
503
|
+
quotaDb
|
|
504
|
+
.query<{ bytes: number }, [string]>(
|
|
505
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
506
|
+
)
|
|
507
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
508
|
+
quotaDb
|
|
509
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
510
|
+
.run(used, reserved, quotaOwner);
|
|
472
511
|
if (used + reserved + bytes > quotaBytes) {
|
|
473
512
|
quotaDb.exec("ROLLBACK");
|
|
474
513
|
return "quota";
|
|
@@ -478,8 +517,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
478
517
|
quotaDb.exec("ROLLBACK");
|
|
479
518
|
return "disk";
|
|
480
519
|
}
|
|
481
|
-
quotaDb
|
|
482
|
-
|
|
520
|
+
quotaDb
|
|
521
|
+
.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)")
|
|
522
|
+
.run(token, quotaOwner, bytes, expires);
|
|
523
|
+
quotaDb
|
|
524
|
+
.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2")
|
|
525
|
+
.run(reserved + bytes, quotaOwner);
|
|
483
526
|
quotaDb.exec("COMMIT");
|
|
484
527
|
return "ok";
|
|
485
528
|
} catch (error) {
|
|
@@ -492,17 +535,24 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
492
535
|
if (!quotaDb || !quotaOwner) return;
|
|
493
536
|
quotaDb.exec("BEGIN IMMEDIATE");
|
|
494
537
|
try {
|
|
495
|
-
const reservation = quotaDb
|
|
538
|
+
const reservation = quotaDb
|
|
539
|
+
.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?")
|
|
540
|
+
.get(token);
|
|
496
541
|
if (!reservation) {
|
|
497
542
|
quotaDb.exec("COMMIT");
|
|
498
543
|
return;
|
|
499
544
|
}
|
|
500
545
|
quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
|
|
501
546
|
const used = committedR2Bytes();
|
|
502
|
-
const reserved =
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
547
|
+
const reserved =
|
|
548
|
+
quotaDb
|
|
549
|
+
.query<{ bytes: number }, [string]>(
|
|
550
|
+
"SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
|
|
551
|
+
)
|
|
552
|
+
.get(quotaOwner)?.bytes ?? 0;
|
|
553
|
+
quotaDb
|
|
554
|
+
.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3")
|
|
555
|
+
.run(used, reserved, quotaOwner);
|
|
506
556
|
quotaDb.exec("COMMIT");
|
|
507
557
|
} catch (error) {
|
|
508
558
|
quotaDb.exec("ROLLBACK");
|
|
@@ -510,7 +560,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
510
560
|
}
|
|
511
561
|
};
|
|
512
562
|
|
|
513
|
-
const recordR2TransferMetric = (
|
|
563
|
+
const recordR2TransferMetric = (
|
|
564
|
+
field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size",
|
|
565
|
+
): void => {
|
|
514
566
|
if (!quotaDb || !quotaOwner) return;
|
|
515
567
|
quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
|
|
516
568
|
};
|
|
@@ -528,19 +580,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
528
580
|
};
|
|
529
581
|
|
|
530
582
|
type KvStmts = {
|
|
531
|
-
get: Statement<
|
|
583
|
+
get: Statement<
|
|
584
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
585
|
+
[string, string]
|
|
586
|
+
>;
|
|
532
587
|
put: Statement<unknown, [string, string, string, number | null, number, string | null]>;
|
|
533
588
|
del: Statement<unknown, [string, string]>;
|
|
534
|
-
list: Statement<
|
|
589
|
+
list: Statement<
|
|
590
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
591
|
+
[string, string, string, number, number]
|
|
592
|
+
>;
|
|
535
593
|
};
|
|
536
594
|
const kvStmtCache = new Map<Database, KvStmts>();
|
|
537
595
|
const kvStmts = (store: Database): KvStmts => {
|
|
538
596
|
let stmts = kvStmtCache.get(store);
|
|
539
597
|
if (!stmts) {
|
|
540
598
|
stmts = {
|
|
541
|
-
get: store.query<
|
|
542
|
-
|
|
543
|
-
|
|
599
|
+
get: store.query<
|
|
600
|
+
{ value: string; expires_at: number | null; binary: number; metadata: string | null },
|
|
601
|
+
[string, string]
|
|
602
|
+
>("SELECT value, expires_at, binary, metadata FROM kv WHERE ns = ? AND key = ?"),
|
|
544
603
|
put: store.query(
|
|
545
604
|
"INSERT INTO kv (ns, key, value, expires_at, binary, metadata) VALUES (?1, ?2, ?3, ?4, ?5, ?6) " +
|
|
546
605
|
"ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5, metadata = ?6",
|
|
@@ -550,7 +609,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
550
609
|
// it is a separate concern (ponytail: rows accumulate until touched by a
|
|
551
610
|
// get/list/put on the same key/prefix, add a cron sweep if that matters).
|
|
552
611
|
// The prefix matches exactly: LIKE ignored case and read "_" and "%" as wildcards.
|
|
553
|
-
list: store.query<
|
|
612
|
+
list: store.query<
|
|
613
|
+
{ key: string; expires_at: number | null; metadata: string | null },
|
|
614
|
+
[string, string, string, number, number]
|
|
615
|
+
>(
|
|
554
616
|
"SELECT key, expires_at, metadata FROM kv WHERE ns = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 " +
|
|
555
617
|
"AND (expires_at IS NULL OR expires_at > ?4) ORDER BY key LIMIT ?5",
|
|
556
618
|
),
|
|
@@ -566,7 +628,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
566
628
|
* seconds) -> ms epoch, or `null` for no expiry. */
|
|
567
629
|
const kvExpiresAt = (msg: Frame): number | null => {
|
|
568
630
|
if (isNumber(msg.expirationTtl)) {
|
|
569
|
-
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
631
|
+
if (msg.expirationTtl < MIN_KV_TTL_SECONDS)
|
|
632
|
+
throw new Error(`expirationTtl must be at least ${MIN_KV_TTL_SECONDS} seconds`);
|
|
570
633
|
return Date.now() + msg.expirationTtl * 1000;
|
|
571
634
|
}
|
|
572
635
|
if (isNumber(msg.expiration)) return msg.expiration * 1000;
|
|
@@ -612,6 +675,27 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
612
675
|
return d1Session ? d1Session.execute(request) : d1Store.execute({ ...request, owner: "dedicated" });
|
|
613
676
|
};
|
|
614
677
|
|
|
678
|
+
// Durable Objects storage.sql (the shim's compatibility date 2026-10-29): one SQLite file per
|
|
679
|
+
// object, `<do-sql>/CLASS/FILE.sqlite`, run as a strict D1 query. The directory is named
|
|
680
|
+
// after the DO store file: several projects keep their doDb side by side.
|
|
681
|
+
// FILE is the object's 64-hex id (or a hash of an older id); the class must be bound.
|
|
682
|
+
// Without a SQL pool, at most 64 stay open; the one used longest ago closes first.
|
|
683
|
+
const doSqlDir = opts.doDb
|
|
684
|
+
? `${resolve(opts.doDb).replace(/\.sqlite$/, "")}.do-sql`
|
|
685
|
+
: inMemory
|
|
686
|
+
? null
|
|
687
|
+
: join(dirname(resolve(dbPath)), "do-sql");
|
|
688
|
+
const doSqlStore = createD1Store(0, "", 64);
|
|
689
|
+
const doSql = (msg: Frame): Frame | Promise<Frame> => {
|
|
690
|
+
const cls = requireDoClass(msg.cls);
|
|
691
|
+
const file = str(msg.file);
|
|
692
|
+
if (!/^[0-9a-f]{64}$/.test(file)) throw new Error("invalid Durable Object SQL database");
|
|
693
|
+
const path = doSqlDir ? resolve(doSqlDir, cls, `${file}.sqlite`) : ":memory:";
|
|
694
|
+
const key = path === ":memory:" ? `${token}:${dbPath}:do-sql:${cls}:${file}` : path;
|
|
695
|
+
const request = { key, path, name: `do-sql:${cls}`, msg: { ...msg, op: "d1.query", strict: true } };
|
|
696
|
+
return d1Session ? d1Session.execute(request) : doSqlStore.execute({ ...request, owner: "dedicated" });
|
|
697
|
+
};
|
|
698
|
+
|
|
615
699
|
/**
|
|
616
700
|
* #56 — an object's bytes never enter SQLite; they live in their own file,
|
|
617
701
|
* named by a hash of bucket+key (so unicode keys and directory traversal
|
|
@@ -761,7 +845,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
761
845
|
});
|
|
762
846
|
};
|
|
763
847
|
|
|
764
|
-
const validateCompletedParts = (
|
|
848
|
+
const validateCompletedParts = (
|
|
849
|
+
stored: R2PartRow[],
|
|
850
|
+
requested: Array<{ partNumber: number; etag: string }>,
|
|
851
|
+
): void => {
|
|
765
852
|
if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
|
|
766
853
|
for (let index = 0; index < stored.length; index++) {
|
|
767
854
|
const actual = stored[index];
|
|
@@ -862,7 +949,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
862
949
|
const path = join(directory, name);
|
|
863
950
|
try {
|
|
864
951
|
const tokenValue = name.slice(0, 24);
|
|
865
|
-
const ticket = store
|
|
952
|
+
const ticket = store
|
|
953
|
+
.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?")
|
|
954
|
+
.get(tokenValue);
|
|
866
955
|
if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
|
|
867
956
|
} catch (error) {
|
|
868
957
|
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
|
|
@@ -884,7 +973,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
884
973
|
}
|
|
885
974
|
const start = Number(match[1]);
|
|
886
975
|
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
887
|
-
if (
|
|
976
|
+
if (
|
|
977
|
+
!Number.isSafeInteger(start) ||
|
|
978
|
+
!Number.isSafeInteger(requestedEnd) ||
|
|
979
|
+
start < 0 ||
|
|
980
|
+
start >= size ||
|
|
981
|
+
requestedEnd < start
|
|
982
|
+
)
|
|
983
|
+
return null;
|
|
888
984
|
return { start, end: Math.min(requestedEnd, size - 1) };
|
|
889
985
|
};
|
|
890
986
|
|
|
@@ -911,11 +1007,16 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
911
1007
|
if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
|
|
912
1008
|
return new Response("transfer ticket expired or already used", { status: 410 });
|
|
913
1009
|
}
|
|
914
|
-
if (
|
|
1010
|
+
if (
|
|
1011
|
+
(ticket.method === "upload" && method !== "PUT") ||
|
|
1012
|
+
(ticket.method === "download" && method !== "GET" && method !== "HEAD")
|
|
1013
|
+
) {
|
|
915
1014
|
return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
|
|
916
1015
|
}
|
|
917
1016
|
if (ticket.method === "download") {
|
|
918
|
-
const row = store
|
|
1017
|
+
const row = store
|
|
1018
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1019
|
+
.get(bucket, ticket.key);
|
|
919
1020
|
if (!row) return new Response("not found", { status: 404 });
|
|
920
1021
|
const etag = `"${row.etag}"`;
|
|
921
1022
|
const noneMatch = request.headers.get("if-none-match");
|
|
@@ -947,7 +1048,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
947
1048
|
if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
|
|
948
1049
|
return new Response("invalid content length", { status: 400 });
|
|
949
1050
|
}
|
|
950
|
-
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1051
|
+
if (Number.isFinite(declared) && declared > ticket.max_bytes)
|
|
1052
|
+
return new Response("upload too large", { status: 413 });
|
|
951
1053
|
const directory = r2BlobDir(store);
|
|
952
1054
|
if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
|
|
953
1055
|
if (activeUploads >= maxActiveUploads) {
|
|
@@ -1000,7 +1102,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1000
1102
|
const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
|
|
1001
1103
|
renameSync(temporary, destination);
|
|
1002
1104
|
const previous = store
|
|
1003
|
-
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1105
|
+
.query<Pick<R2Row, "blob_id" | "size">, [string, string]>(
|
|
1106
|
+
"SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?",
|
|
1107
|
+
)
|
|
1004
1108
|
.get(bucket, ticket.key);
|
|
1005
1109
|
const row: R2Row = {
|
|
1006
1110
|
key: ticket.key,
|
|
@@ -1059,7 +1163,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1059
1163
|
try {
|
|
1060
1164
|
for (const part of parts) {
|
|
1061
1165
|
const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
|
|
1062
|
-
if (!bytes || bytes.byteLength !== part.size)
|
|
1166
|
+
if (!bytes || bytes.byteLength !== part.size)
|
|
1167
|
+
throw new Error(`multipart part ${part.part_number} is missing`);
|
|
1063
1168
|
appendFileSync(temporary, bytes);
|
|
1064
1169
|
hash.update(bytes);
|
|
1065
1170
|
size += bytes.byteLength;
|
|
@@ -1105,7 +1210,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1105
1210
|
// group counts as one entry). Same rules as the embedded transport's __sbR2ListPage.
|
|
1106
1211
|
// ponytail: with a delimiter every key after the cursor is read; an index on the
|
|
1107
1212
|
// delimited prefix if buckets grow past ~100k keys.
|
|
1108
|
-
const r2ListPage = (
|
|
1213
|
+
const r2ListPage = (
|
|
1214
|
+
store: Database,
|
|
1215
|
+
bucket: string,
|
|
1216
|
+
prefix: string,
|
|
1217
|
+
cursor: string,
|
|
1218
|
+
limit: number,
|
|
1219
|
+
delimiter: string,
|
|
1220
|
+
) => {
|
|
1109
1221
|
const rows = store
|
|
1110
1222
|
.query<R2Row, [string, string, string]>(
|
|
1111
1223
|
"SELECT * FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 ORDER BY key" +
|
|
@@ -1230,7 +1342,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1230
1342
|
}
|
|
1231
1343
|
}
|
|
1232
1344
|
|
|
1233
|
-
async function proxyFetch(
|
|
1345
|
+
async function proxyFetch(
|
|
1346
|
+
msg: Frame,
|
|
1347
|
+
signal?: AbortSignal,
|
|
1348
|
+
emit?: FrameEmitter,
|
|
1349
|
+
body?: ReadableStream<Uint8Array>,
|
|
1350
|
+
): Promise<Upstream> {
|
|
1234
1351
|
let url: URL;
|
|
1235
1352
|
try {
|
|
1236
1353
|
url = new URL(str(msg.url));
|
|
@@ -1270,7 +1387,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1270
1387
|
* minus the trip through the network (and therefore minus TLS and the
|
|
1271
1388
|
* default-deny egress rules, which is why this is not `fetch`).
|
|
1272
1389
|
*/
|
|
1273
|
-
async function serviceFetch(
|
|
1390
|
+
async function serviceFetch(
|
|
1391
|
+
msg: Frame,
|
|
1392
|
+
signal?: AbortSignal,
|
|
1393
|
+
emit?: FrameEmitter,
|
|
1394
|
+
body?: ReadableStream<Uint8Array>,
|
|
1395
|
+
): Promise<Upstream> {
|
|
1274
1396
|
const binding = str(msg.binding);
|
|
1275
1397
|
const declared = bindings.services.find((entry) => entry.binding === binding);
|
|
1276
1398
|
if (!declared) throw new Error(`service not bound: ${binding}`);
|
|
@@ -1360,7 +1482,13 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1360
1482
|
// last key), with expirations and metadata. Without one: up to 1,000 keys, as before.
|
|
1361
1483
|
const paged = isNumber(msg.limit);
|
|
1362
1484
|
const limit = paged ? Math.max(0, Math.floor(Number(msg.limit))) : 1000;
|
|
1363
|
-
const rows = kvStmts(store).list.all(
|
|
1485
|
+
const rows = kvStmts(store).list.all(
|
|
1486
|
+
part,
|
|
1487
|
+
str(msg.prefix),
|
|
1488
|
+
paged ? str(msg.cursor) : "",
|
|
1489
|
+
Date.now(),
|
|
1490
|
+
paged ? limit + 1 : limit,
|
|
1491
|
+
);
|
|
1364
1492
|
const page = rows.slice(0, limit);
|
|
1365
1493
|
const complete = !paged || rows.length <= limit;
|
|
1366
1494
|
return {
|
|
@@ -1379,17 +1507,22 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1379
1507
|
const name = str(msg.name) || "default";
|
|
1380
1508
|
const key = str(msg.key);
|
|
1381
1509
|
const row = db
|
|
1382
|
-
.query<
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
>("SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?")
|
|
1510
|
+
.query<{ status: number; headers_json: string; body: string; expires_at: number | null }, [string, string]>(
|
|
1511
|
+
"SELECT status, headers_json, body, expires_at FROM cache WHERE name = ? AND cache_key = ?",
|
|
1512
|
+
)
|
|
1386
1513
|
.get(name, key);
|
|
1387
1514
|
if (!row) return { ok: true, found: false };
|
|
1388
1515
|
if (row.expires_at != null && row.expires_at <= Date.now()) {
|
|
1389
1516
|
db.query("DELETE FROM cache WHERE name = ? AND cache_key = ?").run(name, key);
|
|
1390
1517
|
return { ok: true, found: false };
|
|
1391
1518
|
}
|
|
1392
|
-
return {
|
|
1519
|
+
return {
|
|
1520
|
+
ok: true,
|
|
1521
|
+
found: true,
|
|
1522
|
+
status: row.status,
|
|
1523
|
+
headers: jsonObject(parseJsonValue(row.headers_json)) ?? {},
|
|
1524
|
+
body: row.body,
|
|
1525
|
+
};
|
|
1393
1526
|
}
|
|
1394
1527
|
case "cache.put": {
|
|
1395
1528
|
const name = str(msg.name) || "default";
|
|
@@ -1436,6 +1569,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1436
1569
|
case "d1.exec":
|
|
1437
1570
|
case "d1.backup":
|
|
1438
1571
|
return await d1(requireD1(msg.db), msg);
|
|
1572
|
+
case "do.sql":
|
|
1573
|
+
return await doSql(msg);
|
|
1439
1574
|
|
|
1440
1575
|
case "ratelimit.check": {
|
|
1441
1576
|
// #69 — fixed window; limit/period arrive in the message. Same upsert as
|
|
@@ -1503,9 +1638,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1503
1638
|
const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
|
|
1504
1639
|
const expires = Date.now() + expiresIn * 1000;
|
|
1505
1640
|
const maxBytes =
|
|
1506
|
-
method === "upload"
|
|
1507
|
-
? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
|
|
1508
|
-
: 0;
|
|
1641
|
+
method === "upload" ? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit()) : 0;
|
|
1509
1642
|
const expectedSha256 = str(msg.sha256).toLowerCase();
|
|
1510
1643
|
if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
|
|
1511
1644
|
throw new Error("sha256 must be 64 hexadecimal characters");
|
|
@@ -1565,11 +1698,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1565
1698
|
case "r2.head": {
|
|
1566
1699
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1567
1700
|
const key = str(msg.key);
|
|
1568
|
-
const row = store
|
|
1701
|
+
const row = store
|
|
1702
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1703
|
+
.get(bucket, key);
|
|
1569
1704
|
if (!row) return { ok: true, found: false };
|
|
1570
|
-
const body =
|
|
1571
|
-
|
|
1572
|
-
|
|
1705
|
+
const body =
|
|
1706
|
+
msg.op === "r2.get"
|
|
1707
|
+
? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
|
|
1708
|
+
: undefined;
|
|
1573
1709
|
return { ok: true, found: true, object: r2Row(row), body };
|
|
1574
1710
|
}
|
|
1575
1711
|
case "r2.delete": {
|
|
@@ -1585,7 +1721,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1585
1721
|
case "r2.md5": {
|
|
1586
1722
|
// Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
|
|
1587
1723
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1588
|
-
store
|
|
1724
|
+
store
|
|
1725
|
+
.query("UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4")
|
|
1726
|
+
.run(str(msg.md5), bucket, str(msg.key), str(msg.etag));
|
|
1589
1727
|
return { ok: true };
|
|
1590
1728
|
}
|
|
1591
1729
|
case "r2.list": {
|
|
@@ -1673,10 +1811,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1673
1811
|
return row ? { ok: true, found: true, value: row.value } : { ok: true, found: false };
|
|
1674
1812
|
}
|
|
1675
1813
|
case "do.storage.put":
|
|
1676
|
-
doDb
|
|
1677
|
-
|
|
1678
|
-
"
|
|
1679
|
-
|
|
1814
|
+
doDb
|
|
1815
|
+
.query(
|
|
1816
|
+
"INSERT INTO do_storage (cls, id, key, value) VALUES (?1,?2,?3,?4) " +
|
|
1817
|
+
"ON CONFLICT (cls, id, key) DO UPDATE SET value = ?4",
|
|
1818
|
+
)
|
|
1819
|
+
.run(requireDoClass(msg.cls), str(msg.id), str(msg.key), str(msg.value));
|
|
1680
1820
|
return { ok: true };
|
|
1681
1821
|
case "do.storage.delete": {
|
|
1682
1822
|
const r = doDb
|
|
@@ -1690,11 +1830,20 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1690
1830
|
case "do.storage.list": {
|
|
1691
1831
|
const cls = requireDoClass(msg.cls);
|
|
1692
1832
|
const limit = Math.min(Math.max(Number(msg.limit) || 1000, 1), 10000);
|
|
1833
|
+
// start, startAfter, end and reverse come from the runtime's 2026-10-28 storage.
|
|
1834
|
+
// The prefix match is exact: LIKE also matched _ and % as wildcards, case-insensitively.
|
|
1835
|
+
const bound = (v: JsonValue | undefined): string | null => (v == null ? null : str(v));
|
|
1693
1836
|
const rows = doDb
|
|
1694
|
-
.query<
|
|
1695
|
-
|
|
1837
|
+
.query<
|
|
1838
|
+
{ key: string; value: string },
|
|
1839
|
+
[string, string, string, number, string | null, string | null, string | null]
|
|
1840
|
+
>(
|
|
1841
|
+
"SELECT key, value FROM do_storage WHERE cls = ?1 AND id = ?2 AND substr(key, 1, length(?3)) = ?3" +
|
|
1842
|
+
" AND (?5 IS NULL OR key >= ?5) AND (?6 IS NULL OR key > ?6) AND (?7 IS NULL OR key < ?7)" +
|
|
1843
|
+
(msg.reverse === true ? " ORDER BY key DESC" : " ORDER BY key") +
|
|
1844
|
+
" LIMIT ?4",
|
|
1696
1845
|
)
|
|
1697
|
-
.all(cls, str(msg.id), str(msg.prefix), limit);
|
|
1846
|
+
.all(cls, str(msg.id), str(msg.prefix), limit, bound(msg.start), bound(msg.startAfter), bound(msg.end));
|
|
1698
1847
|
return { ok: true, entries: rows.map((r) => [r.key, r.value]) };
|
|
1699
1848
|
}
|
|
1700
1849
|
|
|
@@ -1702,17 +1851,23 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1702
1851
|
// for that object, and attempts resets because this is a fresh schedule.
|
|
1703
1852
|
case "do.alarm.set": {
|
|
1704
1853
|
const cls = requireDoClass(msg.cls);
|
|
1705
|
-
doDb
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1854
|
+
doDb
|
|
1855
|
+
.transaction(() => {
|
|
1856
|
+
doDb
|
|
1857
|
+
.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=? AND expires<=?")
|
|
1858
|
+
.run(cls, str(msg.id), Date.now());
|
|
1859
|
+
doDb
|
|
1860
|
+
.query(
|
|
1861
|
+
"INSERT INTO do_alarm (cls,id,at,attempts) VALUES (?1,?2,?3,0) ON CONFLICT (cls,id) DO UPDATE SET at=?3,attempts=0",
|
|
1862
|
+
)
|
|
1863
|
+
.run(cls, str(msg.id), Math.trunc(Number(msg.at) || 0));
|
|
1864
|
+
})
|
|
1865
|
+
.immediate();
|
|
1866
|
+
return { ok: true };
|
|
1710
1867
|
}
|
|
1711
1868
|
case "do.alarm.get": {
|
|
1712
1869
|
const row = doDb
|
|
1713
|
-
.query<{ at: number }, [string, string]>(
|
|
1714
|
-
"SELECT at FROM do_alarm WHERE cls = ? AND id = ?",
|
|
1715
|
-
)
|
|
1870
|
+
.query<{ at: number }, [string, string]>("SELECT at FROM do_alarm WHERE cls = ? AND id = ?")
|
|
1716
1871
|
.get(requireDoClass(msg.cls), str(msg.id));
|
|
1717
1872
|
const retry = row
|
|
1718
1873
|
? null
|
|
@@ -1725,12 +1880,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1725
1880
|
}
|
|
1726
1881
|
case "do.alarm.delete": {
|
|
1727
1882
|
const cls = requireDoClass(msg.cls);
|
|
1728
|
-
const deleted = doDb
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1883
|
+
const deleted = doDb
|
|
1884
|
+
.transaction(() => {
|
|
1885
|
+
const pending = doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1886
|
+
const delivery = doDb.query("DELETE FROM do_alarm_delivery WHERE cls=? AND id=?").run(cls, str(msg.id));
|
|
1887
|
+
return pending.changes > 0 || delivery.changes > 0;
|
|
1888
|
+
})
|
|
1889
|
+
.immediate();
|
|
1890
|
+
return { ok: true, deleted };
|
|
1734
1891
|
}
|
|
1735
1892
|
|
|
1736
1893
|
case "assets.get": {
|
|
@@ -1830,7 +1987,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1830
1987
|
if (msg.op === "r2.get") {
|
|
1831
1988
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1832
1989
|
const key = str(msg.key);
|
|
1833
|
-
const row = store
|
|
1990
|
+
const row = store
|
|
1991
|
+
.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
|
|
1992
|
+
.get(bucket, key);
|
|
1834
1993
|
if (!row) return { reply: { ok: true, found: false } };
|
|
1835
1994
|
return {
|
|
1836
1995
|
reply: { ok: true, found: true, object: r2Row(row) },
|
|
@@ -1861,7 +2020,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1861
2020
|
} else if (!hit && assetManifest?.notFound === "404-page") {
|
|
1862
2021
|
hit = readAsset("/404.html");
|
|
1863
2022
|
}
|
|
1864
|
-
if (!hit)
|
|
2023
|
+
if (!hit)
|
|
2024
|
+
return { reply: { ok: true, found: false, status: 404, rules }, bytes: new TextEncoder().encode("Not Found") };
|
|
1865
2025
|
return {
|
|
1866
2026
|
reply: { ok: true, found, status, type: hit.type, hash: hit.hash, rules },
|
|
1867
2027
|
bytes: hit.body,
|
|
@@ -1872,7 +2032,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1872
2032
|
}
|
|
1873
2033
|
|
|
1874
2034
|
/** One wire frame in, one out. See `handleFrame` on the Broker type. */
|
|
1875
|
-
async function handleFrame(
|
|
2035
|
+
async function handleFrame(
|
|
2036
|
+
payload: Buffer,
|
|
2037
|
+
signal?: AbortSignal,
|
|
2038
|
+
emit?: FrameEmitter,
|
|
2039
|
+
body?: ReadableStream<Uint8Array>,
|
|
2040
|
+
): Promise<Buffer> {
|
|
1876
2041
|
if (payload.length === 0 || payload[0] !== FRAME_V1) {
|
|
1877
2042
|
return encodeFrame(await handlePayload(payload.toString("utf8")));
|
|
1878
2043
|
}
|
|
@@ -1916,6 +2081,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1916
2081
|
"d1.exec",
|
|
1917
2082
|
"d1.batch",
|
|
1918
2083
|
"d1.backup",
|
|
2084
|
+
"do.sql",
|
|
1919
2085
|
"ratelimit.check",
|
|
1920
2086
|
"r2.put",
|
|
1921
2087
|
"r2.multipart.create",
|
|
@@ -1983,14 +2149,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1983
2149
|
|
|
1984
2150
|
const timeoutMs = opts.deliveryTimeoutMs ?? 25_000;
|
|
1985
2151
|
const leaseMs = opts.deliveryLeaseMs ?? 30_000;
|
|
1986
|
-
if (!(timeoutMs > 0 && leaseMs > timeoutMs))
|
|
1987
|
-
throw new Error("delivery lease must exceed its timeout");
|
|
2152
|
+
if (!(timeoutMs > 0 && leaseMs > timeoutMs)) throw new Error("delivery lease must exceed its timeout");
|
|
1988
2153
|
let closed = false;
|
|
1989
2154
|
const deliveries = new Set<AbortController>();
|
|
1990
|
-
async function deliverTrigger(
|
|
1991
|
-
kind: "scheduled" | "queue" | "alarm",
|
|
1992
|
-
body: JsonObject,
|
|
1993
|
-
): Promise<Response | null> {
|
|
2155
|
+
async function deliverTrigger(kind: "scheduled" | "queue" | "alarm", body: JsonObject): Promise<Response | null> {
|
|
1994
2156
|
if (!opts.sproutUrl || !dispatchEnabled() || closed) return null;
|
|
1995
2157
|
const abort = new AbortController();
|
|
1996
2158
|
deliveries.add(abort);
|
|
@@ -2007,21 +2169,30 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2007
2169
|
},
|
|
2008
2170
|
body: JSON.stringify(body),
|
|
2009
2171
|
}).then(async (res) => {
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2172
|
+
if (kind !== "queue") {
|
|
2173
|
+
await res.body?.cancel();
|
|
2174
|
+
return new Response(null, { status: res.status });
|
|
2175
|
+
}
|
|
2176
|
+
const chunks: Uint8Array[] = [];
|
|
2177
|
+
let size = 0;
|
|
2178
|
+
const reader = res.body?.getReader();
|
|
2179
|
+
try {
|
|
2180
|
+
if (reader)
|
|
2181
|
+
for (;;) {
|
|
2016
2182
|
const part = await reader.read();
|
|
2017
2183
|
if (part.done) break;
|
|
2018
2184
|
size += part.value.byteLength;
|
|
2019
|
-
if (size > 65_536) {
|
|
2185
|
+
if (size > 65_536) {
|
|
2186
|
+
await reader.cancel();
|
|
2187
|
+
throw new Error("delivery acknowledgement exceeds 64 KiB");
|
|
2188
|
+
}
|
|
2020
2189
|
chunks.push(part.value);
|
|
2021
2190
|
}
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
}
|
|
2191
|
+
} finally {
|
|
2192
|
+
reader?.releaseLock();
|
|
2193
|
+
}
|
|
2194
|
+
return new Response(size ? Buffer.concat(chunks) : null, { status: res.status, headers: res.headers });
|
|
2195
|
+
}),
|
|
2025
2196
|
new Promise<never>((_, reject) =>
|
|
2026
2197
|
abort.signal.addEventListener("abort", () => reject(new Error("delivery interrupted")), {
|
|
2027
2198
|
once: true,
|
|
@@ -2098,9 +2269,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2098
2269
|
for (const id of ack)
|
|
2099
2270
|
if (!retrySet.has(id))
|
|
2100
2271
|
store
|
|
2101
|
-
.query(
|
|
2102
|
-
"DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?",
|
|
2103
|
-
)
|
|
2272
|
+
.query("DELETE FROM mq WHERE queue = ? AND id = ? AND claim_token = ? AND visible_at > ?")
|
|
2104
2273
|
.run(part, id, claim, Date.now());
|
|
2105
2274
|
for (const id of retry)
|
|
2106
2275
|
store
|
|
@@ -2120,7 +2289,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2120
2289
|
const now = Date.now();
|
|
2121
2290
|
const claimed = doDb
|
|
2122
2291
|
.transaction(() => {
|
|
2123
|
-
doDb
|
|
2292
|
+
doDb
|
|
2293
|
+
.query(
|
|
2294
|
+
"DELETE FROM do_alarm_delivery WHERE expires<=? AND attempts>=5 AND EXISTS (SELECT 1 FROM do_alarm a WHERE a.cls=do_alarm_delivery.cls AND a.id=do_alarm_delivery.id)",
|
|
2295
|
+
)
|
|
2296
|
+
.run(now);
|
|
2124
2297
|
const due = doDb
|
|
2125
2298
|
.query<{ cls: string; id: string; at: number; attempts: number }, [number]>(
|
|
2126
2299
|
"SELECT cls,id,at,attempts FROM do_alarm WHERE at <= ? AND NOT EXISTS (SELECT 1 FROM do_alarm_delivery d WHERE d.cls=do_alarm.cls AND d.id=do_alarm.id) ORDER BY at LIMIT 10",
|
|
@@ -2128,9 +2301,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2128
2301
|
.all(now)
|
|
2129
2302
|
.filter((r) => bindings.do.some((b) => b.className === r.cls));
|
|
2130
2303
|
for (const r of due) {
|
|
2131
|
-
doDb
|
|
2132
|
-
.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)")
|
|
2133
|
-
.run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2304
|
+
doDb.query("INSERT INTO do_alarm_delivery VALUES (?,?,?,?,?,?)").run(r.cls, r.id, r.at, r.attempts, "", 0);
|
|
2134
2305
|
doDb.query("DELETE FROM do_alarm WHERE cls=? AND id=? AND at=?").run(r.cls, r.id, r.at);
|
|
2135
2306
|
}
|
|
2136
2307
|
const expired = doDb
|
|
@@ -2142,9 +2313,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2142
2313
|
return expired.map((r) => {
|
|
2143
2314
|
const claim = randomBytes(24).toString("hex");
|
|
2144
2315
|
doDb
|
|
2145
|
-
.query(
|
|
2146
|
-
"UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?",
|
|
2147
|
-
)
|
|
2316
|
+
.query("UPDATE do_alarm_delivery SET token=?,expires=?,attempts=attempts+1 WHERE cls=? AND id=?")
|
|
2148
2317
|
.run(claim, now + leaseMs, r.cls, r.id);
|
|
2149
2318
|
return { ...r, claim };
|
|
2150
2319
|
});
|
|
@@ -2159,9 +2328,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2159
2328
|
.run(r.cls, r.id, r.claim, Date.now());
|
|
2160
2329
|
else
|
|
2161
2330
|
doDb
|
|
2162
|
-
.query(
|
|
2163
|
-
"UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?",
|
|
2164
|
-
)
|
|
2331
|
+
.query("UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires > ?")
|
|
2165
2332
|
.run(Date.now() + 5_000, r.cls, r.id, r.claim, Date.now());
|
|
2166
2333
|
});
|
|
2167
2334
|
}
|
|
@@ -2179,8 +2346,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2179
2346
|
if (stamp === lastTick) return; // once per minute
|
|
2180
2347
|
lastTick = stamp;
|
|
2181
2348
|
for (const expr of bindings.crons) {
|
|
2182
|
-
if (cronMatches(expr, now))
|
|
2183
|
-
void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2349
|
+
if (cronMatches(expr, now)) void deliverTrigger("scheduled", { cron: expr, scheduledTime: now.getTime() });
|
|
2184
2350
|
}
|
|
2185
2351
|
}, 15_000),
|
|
2186
2352
|
);
|
|
@@ -2198,6 +2364,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2198
2364
|
for (const t of timers) clearInterval(t);
|
|
2199
2365
|
d1Session?.close();
|
|
2200
2366
|
d1Store.close();
|
|
2367
|
+
doSqlStore.close();
|
|
2201
2368
|
for (const conn of resourceDbs.values()) conn.close();
|
|
2202
2369
|
if (doDb !== db) doDb.close();
|
|
2203
2370
|
db.close();
|
|
@@ -2344,7 +2511,10 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2344
2511
|
const state = socket.data;
|
|
2345
2512
|
if (state.closed || !state.output) return;
|
|
2346
2513
|
const accepted = socket.write(state.output.subarray(state.offset));
|
|
2347
|
-
if (accepted < 0) {
|
|
2514
|
+
if (accepted < 0) {
|
|
2515
|
+
socket.end();
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2348
2518
|
state.offset += accepted;
|
|
2349
2519
|
if (state.offset === state.output.length) {
|
|
2350
2520
|
state.output = null;
|
|
@@ -2398,47 +2568,56 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2398
2568
|
state.size += chunk.length;
|
|
2399
2569
|
if (state.processing) return;
|
|
2400
2570
|
state.processing = true;
|
|
2401
|
-
try {
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2571
|
+
try {
|
|
2572
|
+
for (;;) {
|
|
2573
|
+
if (state.closed) return;
|
|
2574
|
+
if (state.size < 4) return;
|
|
2575
|
+
if (state.chunks.length > 1) {
|
|
2576
|
+
state.chunks = [Buffer.concat(state.chunks, state.size)];
|
|
2577
|
+
}
|
|
2578
|
+
const buf = state.chunks[0];
|
|
2579
|
+
const len = buf.readUInt32LE(0);
|
|
2580
|
+
if (len > MAX_FRAME) {
|
|
2581
|
+
await send(socket, encodeFrame({ ok: false, error: "frame too large" }));
|
|
2582
|
+
socket.end();
|
|
2583
|
+
return;
|
|
2584
|
+
}
|
|
2585
|
+
if (state.size < 4 + len) return;
|
|
2586
|
+
// Kept as bytes: a v1 frame carries binary after its JSON, and
|
|
2587
|
+
// decoding the whole payload as utf8 would corrupt it.
|
|
2588
|
+
const payload = buf.subarray(4, 4 + len);
|
|
2589
|
+
const rest = buf.subarray(4 + len);
|
|
2590
|
+
state.chunks = rest.length ? [Buffer.from(rest)] : [];
|
|
2591
|
+
state.size = rest.length;
|
|
2592
|
+
// A frame of a streamed request body: to its request's stream.
|
|
2593
|
+
if (payload.length > 0 && payload[0] === FRAME_V1_MORE) {
|
|
2594
|
+
state.sink?.frame(payload);
|
|
2595
|
+
continue;
|
|
2596
|
+
}
|
|
2597
|
+
// A request whose body follows in frames: it runs while this loop feeds them.
|
|
2598
|
+
if (bodyStreamHead(payload)) {
|
|
2599
|
+
const sink = bodySink(socket);
|
|
2600
|
+
state.sink = sink;
|
|
2601
|
+
void (async () => {
|
|
2602
|
+
const reply = await broker.handleFrame(
|
|
2603
|
+
payload,
|
|
2604
|
+
state.abort.signal,
|
|
2605
|
+
(frame) => send(socket, frame),
|
|
2606
|
+
sink.stream,
|
|
2607
|
+
);
|
|
2608
|
+
await send(socket, reply);
|
|
2609
|
+
// Body frames the sprout still sends (the upstream answered early) are dropped.
|
|
2610
|
+
sink.fail("the request already ended");
|
|
2611
|
+
})();
|
|
2612
|
+
continue;
|
|
2613
|
+
}
|
|
2614
|
+
// Bun does not buffer unwritten bytes. Serialize replies and retain
|
|
2615
|
+
// the unsent suffix until drain, including binary R2 responses.
|
|
2616
|
+
await send(socket, await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame)));
|
|
2437
2617
|
}
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
} } finally { state.processing = false; }
|
|
2618
|
+
} finally {
|
|
2619
|
+
state.processing = false;
|
|
2620
|
+
}
|
|
2442
2621
|
},
|
|
2443
2622
|
},
|
|
2444
2623
|
});
|
|
@@ -2449,10 +2628,15 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2449
2628
|
export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
2450
2629
|
const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
|
|
2451
2630
|
const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
|
|
2452
|
-
const server = Bun.serve({
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2631
|
+
const server = Bun.serve({
|
|
2632
|
+
hostname,
|
|
2633
|
+
port,
|
|
2634
|
+
maxRequestBodySize,
|
|
2635
|
+
fetch(request, server) {
|
|
2636
|
+
server.timeout(request, 255);
|
|
2637
|
+
return broker.transfer(request);
|
|
2638
|
+
},
|
|
2639
|
+
});
|
|
2456
2640
|
return { port: server.port ?? port, stop: () => server.stop(true) };
|
|
2457
2641
|
}
|
|
2458
2642
|
|
|
@@ -2498,9 +2682,7 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2498
2682
|
const parsed = jsonObject(parseJsonValue(values.services));
|
|
2499
2683
|
if (parsed) {
|
|
2500
2684
|
services = Object.fromEntries(
|
|
2501
|
-
Object.entries(parsed).flatMap(([binding, host]) =>
|
|
2502
|
-
isString(host) ? [[binding, host] as const] : [],
|
|
2503
|
-
),
|
|
2685
|
+
Object.entries(parsed).flatMap(([binding, host]) => (isString(host) ? [[binding, host] as const] : [])),
|
|
2504
2686
|
);
|
|
2505
2687
|
}
|
|
2506
2688
|
} catch {
|
|
@@ -2518,13 +2700,9 @@ export function startBroker(argv: string[], d1Pool?: D1Pool) {
|
|
|
2518
2700
|
resourceDir: values["resource-dir"],
|
|
2519
2701
|
doDb: values["do-db"],
|
|
2520
2702
|
ownerId: values["owner-id"],
|
|
2521
|
-
r2QuotaBytes:
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
? String(values["r2-resource-ids"]).split(",").filter(Boolean)
|
|
2525
|
-
: undefined,
|
|
2526
|
-
r2MinFreeBytes:
|
|
2527
|
-
Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2703
|
+
r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
|
|
2704
|
+
r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
|
|
2705
|
+
r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
|
|
2528
2706
|
token: values.token ?? process.env.SB_BROKER_TOKEN,
|
|
2529
2707
|
bindings,
|
|
2530
2708
|
secrets,
|