@sproutboat/wire 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/package.json +2 -2
- package/src/broker.test.ts +79 -0
- package/src/broker.ts +162 -13
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,46 @@
|
|
|
1
1
|
# @sproutboat/wire
|
|
2
2
|
|
|
3
|
+
## 0.17.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 829ed12: Request bodies stream, from compatibility date `2026-10-27`, as in workerd:
|
|
8
|
+
|
|
9
|
+
- **Ingress.** The native server runs the handler as soon as a request's headers arrive. `request.body` hands out the bytes as they come in, and the server pauses the socket while more than 1 MiB waits unread, so a large upload read through the stream keeps memory flat (a 128 MiB upload peaks at 38 MiB RSS). `text()`, `json()`, `arrayBuffer()` and `bytes()` read the whole body and return Promises.
|
|
10
|
+
- **Limit.** `SB_REQUEST_BODY_MAX` defaults to 100 MiB for streamed bodies (it stays 1 MiB before the date, where the body is held in memory whole). A larger `Content-Length` gets `413 Payload Too Large` before the handler runs; a chunked body that grows past it closes the connection, and the handler's stream fails.
|
|
11
|
+
- **Service bindings and `fetch()`.** On the broker transport, a `ReadableStream` request body goes out as it is read, in `FRAME_V1_MORE` frames after the request's head, instead of being read to the end first. The broker feeds them to the upstream request as a stream (with backpressure) and says so in its `ping` reply (`bodyStream: true`); against an older broker, bodies are sent whole as before. A streamed body can't follow a 307 or 308 redirect.
|
|
12
|
+
|
|
13
|
+
Internal trigger and RPC calls keep buffered bodies. Before `2026-10-27` nothing changes: the handler gets the whole body at once and `request.text()` answers synchronously, which a deployed handler may rely on, hence the new date.
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- Updated dependencies [829ed12]
|
|
18
|
+
- @sproutboat/runtime@0.28.0
|
|
19
|
+
|
|
20
|
+
## 0.16.0
|
|
21
|
+
|
|
22
|
+
### Minor Changes
|
|
23
|
+
|
|
24
|
+
- 9be9c74: R2 matches workerd from compatibility date `2026-10-26`:
|
|
25
|
+
|
|
26
|
+
- `get`, `put`, `head`, `delete`, `list` and `createMultipartUpload` return Promises, and errors reject them.
|
|
27
|
+
- An object's `etag` is the MD5 of its bytes, with `checksums.md5`. Objects written before keep their old etag.
|
|
28
|
+
- Objects have `uploaded` as a `Date`, `version`, `storageClass`, `range` and `writeHttpMetadata(headers)`. A body has `body` (a `ReadableStream`), `bodyUsed`, `text()`, `json()`, `arrayBuffer()` and `blob()`.
|
|
29
|
+
- `get(key, { range, onlyIf })` takes `{ offset, length }`, `{ suffix }` or a `Range` header, and `etagMatches`, `etagDoesNotMatch`, `uploadedBefore`, `uploadedAfter` or `If-*` headers. When the condition fails it returns the object without a body.
|
|
30
|
+
- `put(key, value, { onlyIf, httpMetadata })` takes a `ReadableStream` or a `Blob`, and `httpMetadata` as `Headers`. When the condition fails it returns `null`.
|
|
31
|
+
- `delete([keys])` deletes several keys.
|
|
32
|
+
- `list({ prefix, limit, cursor, delimiter, include })` pages at up to 1,000 entries, groups keys into `delimitedPrefixes`, and returns `httpMetadata` and `customMetadata` only when `include` asks for them.
|
|
33
|
+
- A value of the wrong type and a key longer than 1,024 bytes fail with R2's errors.
|
|
34
|
+
|
|
35
|
+
Earlier dates keep the synchronous shim. One change applies at every date: **a `list()` prefix now matches exactly.** It used SQL `LIKE`, so it ignored case and read `_` and `%` as wildcards. Both transports store an object's MD5 in a new column, added to existing stores when they open.
|
|
36
|
+
|
|
37
|
+
Against miniflare 4.20260730, `conformance/bindings/r2.js` in sproutd passes 48/48 at `2026-10-26` (16/48 before).
|
|
38
|
+
|
|
39
|
+
### Patch Changes
|
|
40
|
+
|
|
41
|
+
- Updated dependencies [9be9c74]
|
|
42
|
+
- @sproutboat/runtime@0.27.0
|
|
43
|
+
|
|
3
44
|
## 0.15.0
|
|
4
45
|
|
|
5
46
|
### Minor Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sproutboat/wire",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -23,6 +23,6 @@
|
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
25
|
"@sproutboat/assets": "^0.3.0",
|
|
26
|
-
"@sproutboat/runtime": "^0.
|
|
26
|
+
"@sproutboat/runtime": "^0.28.0"
|
|
27
27
|
}
|
|
28
28
|
}
|
package/src/broker.test.ts
CHANGED
|
@@ -424,6 +424,28 @@ test("R2: put / get / head / list / delete on a bound bucket", async () => {
|
|
|
424
424
|
await expect(b.dispatch({ op: "r2.put", bucket: "NOPE", key: "x", body: "y" })).rejects.toThrow("not bound");
|
|
425
425
|
});
|
|
426
426
|
|
|
427
|
+
test("R2: exact list prefix at every date; md5, paged list and delimiter for the shim from 2026-10-26", async () => {
|
|
428
|
+
const b = make({ bindings: { r2: ["ASSETS"] } });
|
|
429
|
+
for (const key of ["a_x", "ab", "AB", "d/1", "d/2", "e/1", "f"]) await b.dispatch({ op: "r2.put", bucket: "ASSETS", key, body: "v" });
|
|
430
|
+
// LIKE listed "ab" for "a_" and "AB" for "a".
|
|
431
|
+
expect(arr((await b.dispatch({ op: "r2.list", bucket: "ASSETS", prefix: "a_" })).objects).map((o) => obj(o).key)).toEqual(["a_x"]);
|
|
432
|
+
expect(arr((await b.dispatch({ op: "r2.list", bucket: "ASSETS", prefix: "a" })).objects).map((o) => obj(o).key)).toEqual(["a_x", "ab"]);
|
|
433
|
+
const put = await b.dispatch({ op: "r2.put", bucket: "ASSETS", key: "m", body: "hello" });
|
|
434
|
+
await b.dispatch({ op: "r2.md5", bucket: "ASSETS", key: "m", etag: obj(put.object).etag, md5: "5d41402abc4b2a76b9719d911017c592" });
|
|
435
|
+
expect(obj((await b.dispatch({ op: "r2.head", bucket: "ASSETS", key: "m" })).object).md5).toBe("5d41402abc4b2a76b9719d911017c592");
|
|
436
|
+
// A stale etag (the object was written again since) records nothing.
|
|
437
|
+
await b.dispatch({ op: "r2.md5", bucket: "ASSETS", key: "m", etag: "stale", md5: "x" });
|
|
438
|
+
expect(obj((await b.dispatch({ op: "r2.head", bucket: "ASSETS", key: "m" })).object).md5).toBe("5d41402abc4b2a76b9719d911017c592");
|
|
439
|
+
const page = (cursor: string) => b.dispatch({ op: "r2.list", bucket: "ASSETS", page: true, prefix: "", cursor, limit: 3, delimiter: "/" });
|
|
440
|
+
const first = await page("");
|
|
441
|
+
expect(arr(first.objects).map((o) => obj(o).key)).toEqual(["AB", "a_x", "ab"]);
|
|
442
|
+
expect(first).toMatchObject({ truncated: true, cursor: "ab", delimitedPrefixes: [] });
|
|
443
|
+
const second = await page("ab");
|
|
444
|
+
expect(arr(second.objects).map((o) => obj(o).key)).toEqual(["f"]);
|
|
445
|
+
expect(second).toMatchObject({ truncated: true, delimitedPrefixes: ["d/", "e/"] });
|
|
446
|
+
expect(await page(String(second.cursor))).toMatchObject({ objects: [{ key: "m" }], truncated: false, cursor: "" });
|
|
447
|
+
});
|
|
448
|
+
|
|
427
449
|
test("R2 direct transfer: streamed upload, metadata, one-use ticket and ranged download", async () => {
|
|
428
450
|
const root = mkdtempSync(join(tmpdir(), "sb-r2-transfer-"));
|
|
429
451
|
try {
|
|
@@ -1515,3 +1537,60 @@ test("R2 direct download: If-None-Match supports wildcard, lists and weak compar
|
|
|
1515
1537
|
expect((await broker.transfer(new Request(url))).status).toBe(410);
|
|
1516
1538
|
} finally { broker.close(); rmSync(dir, { recursive: true, force: true }); }
|
|
1517
1539
|
});
|
|
1540
|
+
|
|
1541
|
+
test("listen(): a service.fetch body streamed in FRAME_V1_MORE frames reaches the upstream as it arrives", async () => {
|
|
1542
|
+
const arrivals: { text: string; at: number }[] = [];
|
|
1543
|
+
const fetchImpl: FetchLike = async (_url, init) => {
|
|
1544
|
+
// SAFETY: the broker passes the BodySink's ReadableStream as the body of a streamed request.
|
|
1545
|
+
const reader = (init?.body as ReadableStream<Uint8Array>).getReader();
|
|
1546
|
+
for (;;) {
|
|
1547
|
+
const { value, done } = await reader.read();
|
|
1548
|
+
if (done) break;
|
|
1549
|
+
arrivals.push({ text: new TextDecoder().decode(value), at: Date.now() });
|
|
1550
|
+
}
|
|
1551
|
+
return new Response(arrivals.map((a) => a.text).join(""));
|
|
1552
|
+
};
|
|
1553
|
+
const b = make({
|
|
1554
|
+
token: "tok",
|
|
1555
|
+
bindings: { services: [{ binding: "PEER", service: "peer" }] },
|
|
1556
|
+
services: { PEER: "peer.example.com" },
|
|
1557
|
+
edgeUrl: "http://127.0.0.1:8080/",
|
|
1558
|
+
fetchImpl,
|
|
1559
|
+
});
|
|
1560
|
+
const server = listen(b, "127.0.0.1", 0);
|
|
1561
|
+
const socket = createConnection({ host: "127.0.0.1", port: server.port });
|
|
1562
|
+
const replies: Buffer[] = [];
|
|
1563
|
+
let pending = Buffer.alloc(0);
|
|
1564
|
+
socket.on("data", (data: Buffer) => {
|
|
1565
|
+
pending = Buffer.concat([pending, data]);
|
|
1566
|
+
while (pending.length >= 4 && pending.length >= 4 + pending.readUInt32LE(0)) {
|
|
1567
|
+
replies.push(pending.subarray(4, 4 + pending.readUInt32LE(0)));
|
|
1568
|
+
pending = pending.subarray(4 + pending.readUInt32LE(0));
|
|
1569
|
+
}
|
|
1570
|
+
});
|
|
1571
|
+
await new Promise((done) => socket.once("connect", done));
|
|
1572
|
+
const head = {
|
|
1573
|
+
v: 1,
|
|
1574
|
+
token: "tok",
|
|
1575
|
+
op: "service.fetch",
|
|
1576
|
+
binding: "PEER",
|
|
1577
|
+
url: "https://peer/up",
|
|
1578
|
+
method: "POST",
|
|
1579
|
+
bodyStream: true,
|
|
1580
|
+
};
|
|
1581
|
+
socket.write(frameOf(encodeV1(head)));
|
|
1582
|
+
socket.write(frameOf(encodeV1({ chunk: true }, new TextEncoder().encode("first "), true)));
|
|
1583
|
+
await Bun.sleep(300);
|
|
1584
|
+
expect(arrivals.map((a) => a.text)).toEqual(["first "]);
|
|
1585
|
+
socket.write(frameOf(encodeV1({ chunk: true }, new TextEncoder().encode("second"), true)));
|
|
1586
|
+
socket.write(frameOf(encodeV1({ end: true }, undefined, true)));
|
|
1587
|
+
for (let i = 0; i < 100 && replies.length === 0; i++) await Bun.sleep(20);
|
|
1588
|
+
const jsonLength = replies[0].readUInt32LE(1);
|
|
1589
|
+
expect(JSON.parse(replies[0].subarray(5, 5 + jsonLength).toString())).toMatchObject({ ok: true, status: 200 });
|
|
1590
|
+
expect(replies[0].subarray(5 + jsonLength).toString()).toBe("first second");
|
|
1591
|
+
expect(arrivals[1].at - arrivals[0].at).toBeGreaterThan(200);
|
|
1592
|
+
// The broker says it takes streamed bodies.
|
|
1593
|
+
expect(await b.dispatch({ op: "ping" })).toMatchObject({ bodyStream: true });
|
|
1594
|
+
socket.destroy();
|
|
1595
|
+
server.stop();
|
|
1596
|
+
});
|
package/src/broker.ts
CHANGED
|
@@ -150,6 +150,7 @@ type R2Row = {
|
|
|
150
150
|
http_json: string;
|
|
151
151
|
custom_json: string;
|
|
152
152
|
blob_id: string | null;
|
|
153
|
+
md5?: string | null;
|
|
153
154
|
};
|
|
154
155
|
type R2UploadRow = {
|
|
155
156
|
key: string;
|
|
@@ -195,9 +196,11 @@ export type Broker = {
|
|
|
195
196
|
* `service.fetch` in flight then stops its upstream request too. With `emit`, a
|
|
196
197
|
* `fetch` or `service.fetch` that asks to stream (`stream: true`) sends its head and
|
|
197
198
|
* each body chunk through it as they arrive (FRAME_V1_MORE frames); the returned
|
|
198
|
-
* frame ends the response.
|
|
199
|
+
* frame ends the response. With `body`, a `fetch` or `service.fetch` whose head asked
|
|
200
|
+
* for it (`bodyStream: true`) sends that stream as its request body: the listener feeds
|
|
201
|
+
* it from the FRAME_V1_MORE frames that follow the head on the same connection.
|
|
199
202
|
*/
|
|
200
|
-
handleFrame(payload: Buffer, signal?: AbortSignal, emit?: FrameEmitter): Promise<Buffer>;
|
|
203
|
+
handleFrame(payload: Buffer, signal?: AbortSignal, emit?: FrameEmitter, body?: ReadableStream<Uint8Array>): Promise<Buffer>;
|
|
201
204
|
/** Serve one reserved direct R2 upload or download request. */
|
|
202
205
|
transfer(request: Request): Promise<Response>;
|
|
203
206
|
close(): void;
|
|
@@ -317,6 +320,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
317
320
|
);
|
|
318
321
|
const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
|
|
319
322
|
if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
|
|
323
|
+
// The MD5 of an object's bytes, its etag from compatibility date 2026-10-26 (r2.md5).
|
|
324
|
+
if (!r2Columns.some((column) => column.name === "md5")) conn.exec("ALTER TABLE r2 ADD COLUMN md5 TEXT");
|
|
320
325
|
conn.exec(
|
|
321
326
|
"CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
322
327
|
"uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
|
|
@@ -1095,6 +1100,38 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1095
1100
|
return r2Row(row);
|
|
1096
1101
|
};
|
|
1097
1102
|
|
|
1103
|
+
// One page of r2.list for the shim from 2026-10-26: at most LIMIT entries after CURSOR,
|
|
1104
|
+
// keys under PREFIX, those holding DELIMITER after it grouped into delimitedPrefixes (a
|
|
1105
|
+
// group counts as one entry). Same rules as the embedded transport's __sbR2ListPage.
|
|
1106
|
+
// ponytail: with a delimiter every key after the cursor is read; an index on the
|
|
1107
|
+
// delimited prefix if buckets grow past ~100k keys.
|
|
1108
|
+
const r2ListPage = (store: Database, bucket: string, prefix: string, cursor: string, limit: number, delimiter: string) => {
|
|
1109
|
+
const rows = store
|
|
1110
|
+
.query<R2Row, [string, string, string]>(
|
|
1111
|
+
"SELECT * FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 ORDER BY key" +
|
|
1112
|
+
(delimiter ? "" : ` LIMIT ${limit + 1}`),
|
|
1113
|
+
)
|
|
1114
|
+
.all(bucket, prefix, cursor);
|
|
1115
|
+
const objects: ReturnType<typeof r2Row>[] = [];
|
|
1116
|
+
const delimitedPrefixes: string[] = [];
|
|
1117
|
+
let last = "";
|
|
1118
|
+
let truncated = false;
|
|
1119
|
+
for (const row of rows) {
|
|
1120
|
+
const at = delimiter ? row.key.indexOf(delimiter, prefix.length) : -1;
|
|
1121
|
+
const group = at < 0 ? "" : row.key.slice(0, at + delimiter.length);
|
|
1122
|
+
if (!group || delimitedPrefixes[delimitedPrefixes.length - 1] !== group) {
|
|
1123
|
+
if (objects.length + delimitedPrefixes.length >= limit) {
|
|
1124
|
+
truncated = true;
|
|
1125
|
+
break;
|
|
1126
|
+
}
|
|
1127
|
+
if (group) delimitedPrefixes.push(group);
|
|
1128
|
+
else objects.push(r2Row(row));
|
|
1129
|
+
}
|
|
1130
|
+
last = row.key;
|
|
1131
|
+
}
|
|
1132
|
+
return { ok: true, objects, delimitedPrefixes, truncated, cursor: truncated ? last : "" };
|
|
1133
|
+
};
|
|
1134
|
+
|
|
1098
1135
|
const r2Row = (r: R2Row) => ({
|
|
1099
1136
|
key: r.key,
|
|
1100
1137
|
size: r.size,
|
|
@@ -1102,6 +1139,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1102
1139
|
uploaded: r.uploaded,
|
|
1103
1140
|
httpMetadata: parseJsonValue(r.http_json),
|
|
1104
1141
|
customMetadata: parseJsonValue(r.custom_json),
|
|
1142
|
+
md5: r.md5 ?? null,
|
|
1105
1143
|
});
|
|
1106
1144
|
|
|
1107
1145
|
/**
|
|
@@ -1192,7 +1230,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1192
1230
|
}
|
|
1193
1231
|
}
|
|
1194
1232
|
|
|
1195
|
-
async function proxyFetch(msg: Frame, signal?: AbortSignal, emit?: FrameEmitter): Promise<Upstream> {
|
|
1233
|
+
async function proxyFetch(msg: Frame, signal?: AbortSignal, emit?: FrameEmitter, body?: ReadableStream<Uint8Array>): Promise<Upstream> {
|
|
1196
1234
|
let url: URL;
|
|
1197
1235
|
try {
|
|
1198
1236
|
url = new URL(str(msg.url));
|
|
@@ -1211,7 +1249,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1211
1249
|
const init: RequestInit = {
|
|
1212
1250
|
method,
|
|
1213
1251
|
headers,
|
|
1214
|
-
body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
|
|
1252
|
+
body: method === "GET" || method === "HEAD" ? undefined : (body ?? upstreamBody(msg)),
|
|
1215
1253
|
redirect: "manual",
|
|
1216
1254
|
signal,
|
|
1217
1255
|
};
|
|
@@ -1232,7 +1270,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1232
1270
|
* minus the trip through the network (and therefore minus TLS and the
|
|
1233
1271
|
* default-deny egress rules, which is why this is not `fetch`).
|
|
1234
1272
|
*/
|
|
1235
|
-
async function serviceFetch(msg: Frame, signal?: AbortSignal, emit?: FrameEmitter): Promise<Upstream> {
|
|
1273
|
+
async function serviceFetch(msg: Frame, signal?: AbortSignal, emit?: FrameEmitter, body?: ReadableStream<Uint8Array>): Promise<Upstream> {
|
|
1236
1274
|
const binding = str(msg.binding);
|
|
1237
1275
|
const declared = bindings.services.find((entry) => entry.binding === binding);
|
|
1238
1276
|
if (!declared) throw new Error(`service not bound: ${binding}`);
|
|
@@ -1266,7 +1304,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1266
1304
|
const res = await doFetch(target, {
|
|
1267
1305
|
method,
|
|
1268
1306
|
headers,
|
|
1269
|
-
body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
|
|
1307
|
+
body: method === "GET" || method === "HEAD" ? undefined : (body ?? upstreamBody(msg)),
|
|
1270
1308
|
redirect: "manual",
|
|
1271
1309
|
signal,
|
|
1272
1310
|
});
|
|
@@ -1279,7 +1317,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1279
1317
|
async function dispatch(msg: Frame): Promise<Frame> {
|
|
1280
1318
|
switch (msg.op) {
|
|
1281
1319
|
case "ping":
|
|
1282
|
-
|
|
1320
|
+
// bodyStream: this broker takes streamed request bodies (FRAME_V1_MORE frames after a head).
|
|
1321
|
+
return { ok: true, op: "pong", echo: msg.msg ?? null, pid: process.pid, bodyStream: true };
|
|
1283
1322
|
case "kv.get": {
|
|
1284
1323
|
const { store, part } = storeFor("kv", requireKv(msg.ns));
|
|
1285
1324
|
const stmts = kvStmts(store);
|
|
@@ -1543,14 +1582,22 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1543
1582
|
if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
|
|
1544
1583
|
return { ok: true };
|
|
1545
1584
|
}
|
|
1585
|
+
case "r2.md5": {
|
|
1586
|
+
// Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
|
|
1587
|
+
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1588
|
+
store.query("UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4").run(str(msg.md5), bucket, str(msg.key), str(msg.etag));
|
|
1589
|
+
return { ok: true };
|
|
1590
|
+
}
|
|
1546
1591
|
case "r2.list": {
|
|
1592
|
+
// The prefix matches exactly at every date: LIKE ignored case and read "_" and "%" as wildcards.
|
|
1547
1593
|
const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
|
|
1548
1594
|
const prefix = str(msg.prefix);
|
|
1549
1595
|
const cursor = str(msg.cursor);
|
|
1550
|
-
const limit = Math.min(Math.max(Number(msg.limit) || 1000, 1), 1000);
|
|
1596
|
+
const limit = Math.min(Math.max(Math.floor(Number(msg.limit)) || 1000, 1), 1000);
|
|
1597
|
+
if (msg.page === true) return r2ListPage(store, bucket, prefix, cursor, limit, str(msg.delimiter));
|
|
1551
1598
|
const rows = store
|
|
1552
1599
|
.query<R2Row, [string, string, string, number]>(
|
|
1553
|
-
"SELECT * FROM r2 WHERE bucket = ? AND key
|
|
1600
|
+
"SELECT * FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 ORDER BY key LIMIT ?4",
|
|
1554
1601
|
)
|
|
1555
1602
|
.all(bucket, prefix, cursor, limit + 1);
|
|
1556
1603
|
const truncated = rows.length > limit;
|
|
@@ -1741,6 +1788,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1741
1788
|
binary: Uint8Array,
|
|
1742
1789
|
signal?: AbortSignal,
|
|
1743
1790
|
emit?: FrameEmitter,
|
|
1791
|
+
body?: ReadableStream<Uint8Array>,
|
|
1744
1792
|
): Promise<{ reply: Frame; bytes?: Uint8Array }> {
|
|
1745
1793
|
if (msg.op === "r2.put") {
|
|
1746
1794
|
// The body arrived as bytes; write it straight to its own file rather
|
|
@@ -1791,8 +1839,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1791
1839
|
}
|
|
1792
1840
|
|
|
1793
1841
|
// #232 — a v1 fetch gets the upstream body as raw bytes, not decoded text.
|
|
1794
|
-
if (msg.op === "fetch") return proxyFetch(msg, signal, emit);
|
|
1795
|
-
if (msg.op === "service.fetch") return serviceFetch(msg, signal, emit);
|
|
1842
|
+
if (msg.op === "fetch") return proxyFetch(msg, signal, emit, body);
|
|
1843
|
+
if (msg.op === "service.fetch") return serviceFetch(msg, signal, emit, body);
|
|
1796
1844
|
|
|
1797
1845
|
if (msg.op === "assets.get") {
|
|
1798
1846
|
if (!bindings.assets) throw new Error("assets not bound");
|
|
@@ -1824,7 +1872,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1824
1872
|
}
|
|
1825
1873
|
|
|
1826
1874
|
/** One wire frame in, one out. See `handleFrame` on the Broker type. */
|
|
1827
|
-
async function handleFrame(payload: Buffer, signal?: AbortSignal, emit?: FrameEmitter): Promise<Buffer> {
|
|
1875
|
+
async function handleFrame(payload: Buffer, signal?: AbortSignal, emit?: FrameEmitter, body?: ReadableStream<Uint8Array>): Promise<Buffer> {
|
|
1828
1876
|
if (payload.length === 0 || payload[0] !== FRAME_V1) {
|
|
1829
1877
|
return encodeFrame(await handlePayload(payload.toString("utf8")));
|
|
1830
1878
|
}
|
|
@@ -1840,7 +1888,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1840
1888
|
const seen = replayed.get(replayKey);
|
|
1841
1889
|
if (seen) return frameOf(encodeV1(seen));
|
|
1842
1890
|
}
|
|
1843
|
-
const { reply, bytes } = await dispatchBinary(msg, binary, signal, emit);
|
|
1891
|
+
const { reply, bytes } = await dispatchBinary(msg, binary, signal, emit, body);
|
|
1844
1892
|
if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
|
|
1845
1893
|
return frameOf(encodeV1(reply, bytes));
|
|
1846
1894
|
} catch (e) {
|
|
@@ -1876,6 +1924,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1876
1924
|
"r2.multipart.abort",
|
|
1877
1925
|
"r2.transfer.create",
|
|
1878
1926
|
"r2.delete",
|
|
1927
|
+
"r2.md5",
|
|
1879
1928
|
"do.storage.put",
|
|
1880
1929
|
"do.storage.delete",
|
|
1881
1930
|
"do.storage.delete_all",
|
|
@@ -2193,6 +2242,85 @@ export function encodeFrame(obj: Frame): Buffer {
|
|
|
2193
2242
|
/** The bound TCP listener: the port actually assigned, and its shutdown handle. */
|
|
2194
2243
|
export type BrokerServer = { port: number; stop(): void };
|
|
2195
2244
|
|
|
2245
|
+
/** Whether a payload is the head of a fetch whose request body follows in FRAME_V1_MORE frames. */
|
|
2246
|
+
function bodyStreamHead(payload: Buffer): boolean {
|
|
2247
|
+
if (payload.length < 5 || payload[0] !== FRAME_V1) return false;
|
|
2248
|
+
const jsonLen = payload.readUInt32LE(1);
|
|
2249
|
+
if (5 + jsonLen > payload.length) return false;
|
|
2250
|
+
const head = payload.subarray(5, 5 + jsonLen).toString("utf8");
|
|
2251
|
+
// Cheap first: most frames are not this.
|
|
2252
|
+
if (!head.includes('"bodyStream":true')) return false;
|
|
2253
|
+
const msg = jsonObject(parseJsonValue(head));
|
|
2254
|
+
return msg?.bodyStream === true;
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
/** The request body of a streamed-body request, fed by the frames after its head. */
|
|
2258
|
+
type BodySink = {
|
|
2259
|
+
stream: ReadableStream<Uint8Array>;
|
|
2260
|
+
/** A FRAME_V1_MORE payload: `{ chunk: true }` plus bytes, `{ end: true }`, or `{ error }`. */
|
|
2261
|
+
frame(payload: Buffer): void;
|
|
2262
|
+
fail(reason: string): void;
|
|
2263
|
+
};
|
|
2264
|
+
/** Bytes a streamed request body may hold unread before the sprout's connection is paused. */
|
|
2265
|
+
const BODY_SINK_HIGH = 4 * 1024 * 1024;
|
|
2266
|
+
function bodySink(socket: { pause(): void; resume(): void }): BodySink {
|
|
2267
|
+
let controller: ReadableStreamDefaultController<Uint8Array> | null = null;
|
|
2268
|
+
let finished = false;
|
|
2269
|
+
let paused = false;
|
|
2270
|
+
const resume = () => {
|
|
2271
|
+
if (paused) {
|
|
2272
|
+
paused = false;
|
|
2273
|
+
socket.resume();
|
|
2274
|
+
}
|
|
2275
|
+
};
|
|
2276
|
+
const finish = () => {
|
|
2277
|
+
finished = true;
|
|
2278
|
+
resume();
|
|
2279
|
+
};
|
|
2280
|
+
const stream = new ReadableStream<Uint8Array>(
|
|
2281
|
+
{
|
|
2282
|
+
start(c) {
|
|
2283
|
+
controller = c;
|
|
2284
|
+
},
|
|
2285
|
+
pull() {
|
|
2286
|
+
resume();
|
|
2287
|
+
},
|
|
2288
|
+
cancel() {
|
|
2289
|
+
finish();
|
|
2290
|
+
},
|
|
2291
|
+
},
|
|
2292
|
+
new ByteLengthQueuingStrategy({ highWaterMark: BODY_SINK_HIGH }),
|
|
2293
|
+
);
|
|
2294
|
+
return {
|
|
2295
|
+
stream,
|
|
2296
|
+
frame(payload) {
|
|
2297
|
+
if (finished || !controller || payload.length < 5) return;
|
|
2298
|
+
const jsonLen = payload.readUInt32LE(1);
|
|
2299
|
+
const msg = jsonObject(parseJsonValue(payload.subarray(5, 5 + jsonLen).toString("utf8")));
|
|
2300
|
+
if (msg?.chunk === true) {
|
|
2301
|
+
const bytes = payload.subarray(5 + jsonLen);
|
|
2302
|
+
if (bytes.length) controller.enqueue(new Uint8Array(bytes));
|
|
2303
|
+
if ((controller.desiredSize ?? 1) <= 0 && !paused) {
|
|
2304
|
+
paused = true;
|
|
2305
|
+
socket.pause();
|
|
2306
|
+
}
|
|
2307
|
+
return;
|
|
2308
|
+
}
|
|
2309
|
+
finish();
|
|
2310
|
+
if (msg?.end === true) controller.close();
|
|
2311
|
+
else {
|
|
2312
|
+
const error = msg?.error;
|
|
2313
|
+
controller.error(new Error(isString(error) ? error : "the request body failed"));
|
|
2314
|
+
}
|
|
2315
|
+
},
|
|
2316
|
+
fail(reason) {
|
|
2317
|
+
if (finished) return;
|
|
2318
|
+
finish();
|
|
2319
|
+
controller?.error(new Error(reason));
|
|
2320
|
+
},
|
|
2321
|
+
};
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2196
2324
|
/** Start the TCP listener. Returns the bound port. */
|
|
2197
2325
|
export function listen(broker: Broker, hostname: string, port: number): BrokerServer {
|
|
2198
2326
|
// Chunks are held in a list and joined once, when a whole frame has arrived.
|
|
@@ -2209,6 +2337,8 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2209
2337
|
written: (() => void) | null;
|
|
2210
2338
|
/** Aborted when the connection closes, so upstream requests it started stop. */
|
|
2211
2339
|
abort: AbortController;
|
|
2340
|
+
/** The request body a streamed-body request on this connection is reading, if any. */
|
|
2341
|
+
sink: BodySink | null;
|
|
2212
2342
|
};
|
|
2213
2343
|
const flush = (socket: Bun.Socket<SocketState>): void => {
|
|
2214
2344
|
const state = socket.data;
|
|
@@ -2246,6 +2376,7 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2246
2376
|
offset: 0,
|
|
2247
2377
|
written: null,
|
|
2248
2378
|
abort: new AbortController(),
|
|
2379
|
+
sink: null,
|
|
2249
2380
|
};
|
|
2250
2381
|
},
|
|
2251
2382
|
drain: flush,
|
|
@@ -2256,6 +2387,7 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2256
2387
|
state.chunks = [];
|
|
2257
2388
|
state.size = 0;
|
|
2258
2389
|
state.output = null;
|
|
2390
|
+
state.sink?.fail("the sprout closed the connection");
|
|
2259
2391
|
const done = state.written;
|
|
2260
2392
|
state.written = null;
|
|
2261
2393
|
done?.();
|
|
@@ -2286,6 +2418,23 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
|
|
|
2286
2418
|
const rest = buf.subarray(4 + len);
|
|
2287
2419
|
state.chunks = rest.length ? [Buffer.from(rest)] : [];
|
|
2288
2420
|
state.size = rest.length;
|
|
2421
|
+
// A frame of a streamed request body: to its request's stream.
|
|
2422
|
+
if (payload.length > 0 && payload[0] === FRAME_V1_MORE) {
|
|
2423
|
+
state.sink?.frame(payload);
|
|
2424
|
+
continue;
|
|
2425
|
+
}
|
|
2426
|
+
// A request whose body follows in frames: it runs while this loop feeds them.
|
|
2427
|
+
if (bodyStreamHead(payload)) {
|
|
2428
|
+
const sink = bodySink(socket);
|
|
2429
|
+
state.sink = sink;
|
|
2430
|
+
void (async () => {
|
|
2431
|
+
const reply = await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame), sink.stream);
|
|
2432
|
+
await send(socket, reply);
|
|
2433
|
+
// Body frames the sprout still sends (the upstream answered early) are dropped.
|
|
2434
|
+
sink.fail("the request already ended");
|
|
2435
|
+
})();
|
|
2436
|
+
continue;
|
|
2437
|
+
}
|
|
2289
2438
|
// Bun does not buffer unwritten bytes. Serialize replies and retain
|
|
2290
2439
|
// the unsent suffix until drain, including binary R2 responses.
|
|
2291
2440
|
await send(socket, await broker.handleFrame(payload, state.abort.signal, (frame) => send(socket, frame)));
|