@sproutboat/wire 0.9.0 → 0.9.2

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @sproutboat/wire
 
+## 0.9.2
+
+### Patch Changes
+
+- 733bd9e: Outbound `fetch()` and service-binding responses keep binary bodies intact (baronunread/sproutboat#232). `arrayBuffer()` and `bytes()` on a fetched response used to go through Porffor's `text()`, which the runtime overrides to decode UTF-8, so binary bodies collapsed; they now read the raw bytes. Through the broker, `fetch` and `service.fetch` sent over a v1 frame now get the upstream body as raw bytes in the frame's binary section instead of `TextDecoder` output. v0 frames, which already-deployed sprouts use, are unchanged, and a new sprout talking to an older broker falls back to the previous text behaviour.
+
+## 0.9.1
+
+### Patch Changes
+
+- ed9640a: Point Porffor patch diagnostics, runtime comments, and historical notes to the
+  shared upstream notes in the Sproutboat platform repository.
+
 ## 0.9.0
 
 ### Minor Changes
@@ -62,7 +75,7 @@
   
   Verified end-to-end against both transports (kitchen-sink's full conformance suite, 24/24 standalone + 27/27 broker) and with new unit tests for file-backed persistence, the `:memory:` fallback, and a real binary round-trip over the v1 wire protocol.
   
-  **Known limitation found while verifying this, not fixed here:** a binary `put()` body (bytes ≥ 0x80) is corrupted on the way *in*, before it ever reaches storage — `porf_native_fetch_read_value` (the function every inline-C string read in the runtime goes through) always UTF-8-encodes a Latin-1-range string on read, which is correct for genuine text but wrong for opaque bytes coming off a raw HTTP body. This predates this change (the old SQLite-blob path had the identical corruption) and is not something the storage backend can fix — it needs an input-side counterpart to `#176`'s `x-sb-raw-body` marker, or real `ArrayBuffer`/`Uint8Array` body support (the upstream ask already filed as Draft G in `patches/UPSTREAM.md`).
+  **Known limitation found while verifying this, not fixed here:** a binary `put()` body (bytes ≥ 0x80) is corrupted on the way *in*, before it ever reaches storage: `porf_native_fetch_read_value` (the function every inline-C string read in the runtime goes through) always UTF-8-encodes a Latin-1-range string on read, which is correct for genuine text but wrong for opaque bytes coming off a raw HTTP body. This predates this change (the old SQLite-blob path had the identical corruption) and is not something the storage backend can fix. It needs an input-side counterpart to `#176`'s `x-sb-raw-body` marker, or real `ArrayBuffer`/`Uint8Array` body support (the upstream ask was filed as Porffor #386 and later retracted; see `https://github.com/baronunread/sproutboat/blob/main/patches/upstream/request-body-report-386.md`).
 
 ## 0.5.0
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/wire",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/broker.test.ts

@@ -154,6 +154,23 @@ test("fetch is gated by the exact-host allowlist", async () => {
   expect(calls).toEqual(["https://api.example.com/x"]);
 });
 
+test("#232: a v1 fetch returns the upstream body as raw bytes; v0 keeps decoded text", async () => {
+  const binary = Uint8Array.from({ length: 256 }, (_, i) => i);
+  const fetchImpl: FetchLike = async (url) =>
+    new Response(String(url).endsWith("/bin") ? binary : "café", { status: 200 });
+  const b = make({ bindings: { outbound: ["api.example.com"] }, fetchImpl });
+
+  const raw = decodeV1Frame(
+    await b.handleFrame(encodeV1({ v: 1, token: "tok", op: "fetch", url: "https://api.example.com/bin" })),
+  );
+  expect(raw.json).toMatchObject({ ok: true, status: 200 });
+  expect(raw.json.body).toBeUndefined();
+  expect(Buffer.from(raw.bytes).equals(Buffer.from(binary))).toBe(true);
+
+  // Already-deployed sprouts speak v0 and expect the body as text in the JSON.
+  expect(await b.dispatch({ op: "fetch", url: "https://api.example.com/t" })).toMatchObject({ body: "café" });
+});
+
 test("D1: query / run / batch on a bound database, isolated per name", async () => {
   const b = make({ bindings: { d1: ["DB", "OTHER"] } });
   await b.dispatch({ op: "d1.exec", db: "DB", sql: "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)" });

package/src/broker.ts

@@ -1119,12 +1119,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
    * end the process. 32 MiB by default, and the same variable the embedded
    * transport reads so both backends agree.
    */
-  async function readCapped(response: Response, host: string): Promise<string> {
+  async function readCapped(response: Response, host: string): Promise<Buffer> {
     const cap = Number(process.env.SB_FETCH_MAX_BYTES) || 32 * 1024 * 1024;
     const declared = Number(response.headers.get("content-length") || 0);
     if (declared > cap) throw new Error(`response exceeds SB_FETCH_MAX_BYTES from ${host}`);
     const reader = response.body?.getReader();
-    if (!reader) return "";
+    if (!reader) return Buffer.alloc(0);
     const parts: Uint8Array[] = [];
     let total = 0;
     for (;;) {
@@ -1137,10 +1137,19 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       }
       parts.push(value);
     }
-    return new TextDecoder().decode(Buffer.concat(parts));
+    return Buffer.concat(parts);
   }
 
-  async function proxyFetch(msg: Frame): Promise<Frame> {
+  /**
+   * #232 — an upstream response as metadata plus its raw bytes. The v0 frame
+   * carries the body as UTF-8 text inside the JSON (what already-deployed
+   * sprouts expect), which turns a binary body into U+FFFD soup; the v1 frame
+   * sends the bytes untouched in its binary section instead.
+   */
+  type Upstream = { reply: Frame; bytes: Buffer };
+  const asTextFrame = (u: Upstream) => ({ ...u.reply, body: new TextDecoder().decode(u.bytes) });
+
+  async function proxyFetch(msg: Frame): Promise<Upstream> {
     let url: URL;
     try {
       url = new URL(str(msg.url));
@@ -1165,7 +1174,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     });
     const outHeaders: Array<[string, string]> = [];
     res.headers.forEach((v, k) => outHeaders.push([k, v]));
-    return { ok: true, status: res.status, headers: outHeaders, body: await readCapped(res, url.host) };
+    return { reply: { ok: true, status: res.status, headers: outHeaders }, bytes: await readCapped(res, url.host) };
   }
 
   /**
@@ -1176,7 +1185,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
    * minus the trip through the network (and therefore minus TLS and the
    * default-deny egress rules, which is why this is not `fetch`).
    */
-  async function serviceFetch(msg: Frame): Promise<Frame> {
+  async function serviceFetch(msg: Frame): Promise<Upstream> {
     const binding = str(msg.binding);
     const declared = bindings.services.find((entry) => entry.binding === binding);
     if (!declared) throw new Error(`service not bound: ${binding}`);
@@ -1211,7 +1220,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     });
     const outHeaders: Array<[string, string]> = [];
     res.headers.forEach((v, k) => outHeaders.push([k, v]));
-    return { ok: true, status: res.status, headers: outHeaders, body: await readCapped(res, host) };
+    return { reply: { ok: true, status: res.status, headers: outHeaders }, bytes: await readCapped(res, host) };
   }
 
   async function dispatch(msg: Frame): Promise<Frame> {
@@ -1299,10 +1308,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
         return { ok: true, value: secrets[name] };
       }
       case "fetch":
-        return proxyFetch(msg);
+        return asTextFrame(await proxyFetch(msg));
 
       case "service.fetch":
-        return serviceFetch(msg);
+        return asTextFrame(await serviceFetch(msg));
 
       case "d1.query": {
         const conn = d1(requireD1(msg.db));
@@ -1695,6 +1704,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       };
     }
 
+    // #232 — a v1 fetch gets the upstream body as raw bytes, not decoded text.
+    if (msg.op === "fetch") return proxyFetch(msg);
+    if (msg.op === "service.fetch") return serviceFetch(msg);
+
     if (msg.op === "assets.get") {
       if (!bindings.assets) throw new Error("assets not bound");
       const reqPath = str(msg.path) || "/";