@sproutboat/wire 0.9.1 → 0.9.3

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @sproutboat/wire
 
+## 0.9.3
+
+### Patch Changes
+
+- 5dba469: Binary data survives KV and outbound `fetch()` bodies (baronunread/sproutboat#233, and the request half of #232).
+  - **KV:** `put()` of an `ArrayBuffer` or any typed array stores its bytes, flagged binary, instead of `String(view)` (`"0,1,2,..."`). `get(key, type | { type })` supports `"text"`, `"json"` and `"arrayBuffer"` as in Workers: a binary value read as text decodes as UTF-8, and a text value read as `arrayBuffer` is its UTF-8 encoding. `"stream"` throws a clear error. Existing stores gain the `binary` column on first use.
+  - **`fetch()` and service-binding request bodies:** an `ArrayBuffer` or view is sent as its exact bytes in both transports, instead of `"0,1,2,..."`. The standalone HTTP client also stopped truncating bodies at the first NUL byte.
+
+## 0.9.2
+
+### Patch Changes
+
+- 733bd9e: Outbound `fetch()` and service-binding responses keep binary bodies intact (baronunread/sproutboat#232). `arrayBuffer()` and `bytes()` on a fetched response used to go through Porffor's `text()`, which the runtime overrides to decode UTF-8, so binary bodies collapsed; they now read the raw bytes. Through the broker, `fetch` and `service.fetch` sent over a v1 frame now get the upstream body as raw bytes in the frame's binary section instead of `TextDecoder` output. v0 frames, which already-deployed sprouts use, are unchanged, and a new sprout talking to an older broker falls back to the previous text behaviour.
+
 ## 0.9.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/wire",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/broker.test.ts

@@ -39,12 +39,25 @@ test("ping round-trips", async () => {
   expect(await b.dispatch({ op: "ping", msg: "hi" })).toMatchObject({ ok: true, op: "pong", echo: "hi" });
 });
 
+test("#233: a KV value put as bytes is flagged binary and survives an existing store", async () => {
+  const b = make({ bindings: { kv: ["CACHE"] } });
+  // One char per byte, as the runtime sends a put of an ArrayBuffer/view.
+  const bytes = Array.from({ length: 256 }, (_, i) => String.fromCharCode(i)).join("");
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "bin", value: bytes, binary: true });
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "txt", value: "café" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "bin" })).toEqual({ ok: true, found: true, value: bytes, binary: true });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "txt" })).toMatchObject({ value: "café", binary: false });
+  // Overwriting a binary value with text clears the flag.
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "bin", value: "now text" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "bin" })).toMatchObject({ value: "now text", binary: false });
+});
+
 test("KV put / get / list / delete, scoped to a bound namespace", async () => {
   const b = make({ bindings: { kv: ["CACHE"] } });
   expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: false, value: null });
   await b.dispatch({ op: "kv.put", ns: "CACHE", key: "k", value: "v" });
   await b.dispatch({ op: "kv.put", ns: "CACHE", key: "k2", value: "v2" });
-  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: true, value: "v" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: true, value: "v", binary: false });
   expect(await b.dispatch({ op: "kv.list", ns: "CACHE", prefix: "k" })).toEqual({ ok: true, keys: ["k", "k2"] });
   await b.dispatch({ op: "kv.delete", ns: "CACHE", key: "k" });
   expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toMatchObject({ found: false });
@@ -154,6 +167,38 @@ test("fetch is gated by the exact-host allowlist", async () => {
   expect(calls).toEqual(["https://api.example.com/x"]);
 });
 
+test("#232: a binary fetch request body is sent as its bytes, a text body as UTF-8", async () => {
+  const seen: Uint8Array[] = [];
+  const fetchImpl: FetchLike = async (_url, init) => {
+    seen.push(new Uint8Array(await new Response(init?.body).arrayBuffer()));
+    return new Response("ok");
+  };
+  const b = make({ bindings: { outbound: ["api.example.com"] }, fetchImpl });
+  // The runtime sends a binary body one char per byte, flagged bodyBinary.
+  const raw = Array.from({ length: 256 }, (_, i) => String.fromCharCode(i)).join("");
+  await b.dispatch({ op: "fetch", url: "https://api.example.com/up", method: "POST", body: raw, bodyBinary: true });
+  await b.dispatch({ op: "fetch", url: "https://api.example.com/up", method: "POST", body: "café" });
+  expect(Array.from(seen[0])).toEqual(Array.from({ length: 256 }, (_, i) => i));
+  expect(Array.from(seen[1])).toEqual([99, 97, 102, 195, 169]);
+});
+
+test("#232: a v1 fetch returns the upstream body as raw bytes; v0 keeps decoded text", async () => {
+  const binary = Uint8Array.from({ length: 256 }, (_, i) => i);
+  const fetchImpl: FetchLike = async (url) =>
+    new Response(String(url).endsWith("/bin") ? binary : "café", { status: 200 });
+  const b = make({ bindings: { outbound: ["api.example.com"] }, fetchImpl });
+
+  const raw = decodeV1Frame(
+    await b.handleFrame(encodeV1({ v: 1, token: "tok", op: "fetch", url: "https://api.example.com/bin" })),
+  );
+  expect(raw.json).toMatchObject({ ok: true, status: 200 });
+  expect(raw.json.body).toBeUndefined();
+  expect(Buffer.from(raw.bytes).equals(Buffer.from(binary))).toBe(true);
+
+  // Already-deployed sprouts speak v0 and expect the body as text in the JSON.
+  expect(await b.dispatch({ op: "fetch", url: "https://api.example.com/t" })).toMatchObject({ body: "café" });
+});
+
 test("D1: query / run / batch on a bound database, isolated per name", async () => {
   const b = make({ bindings: { d1: ["DB", "OTHER"] } });
   await b.dispatch({ op: "d1.exec", db: "DB", sql: "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)" });
@@ -688,6 +733,7 @@ test("resource-backed KV persists across brokers and is shared by id", async ()
     expect(await second.dispatch({ op: "kv.get", ns: "SHORTENER", key: "a" })).toEqual({
       ok: true,
       found: true,
+      binary: false,
       value: "1",
     });
     second.close();

package/src/broker.ts

@@ -269,6 +269,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     );
     const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
     if (!kvColumns.some((column) => column.name === "expires_at")) conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
+    // #233 — 1 when the value was put as bytes (stored one char per byte), so get()
+    // can hand back exact bytes or decoded text depending on the requested type.
+    if (!kvColumns.some((column) => column.name === "binary")) conn.exec("ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0");
     // #56 — no `body` column: an object's bytes live in their own file under
     // `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
     // Doubling every object through SQLite's own page cache on top of the
@@ -481,8 +484,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   };
 
   type KvStmts = {
-    get: Statement<{ value: string; expires_at: number | null }, [string, string]>;
-    put: Statement<unknown, [string, string, string, number | null]>;
+    get: Statement<{ value: string; expires_at: number | null; binary: number }, [string, string]>;
+    put: Statement<unknown, [string, string, string, number | null, number]>;
     del: Statement<unknown, [string, string]>;
     list: Statement<{ key: string }, [string, string, number]>;
   };
@@ -491,12 +494,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     let stmts = kvStmtCache.get(store);
     if (!stmts) {
       stmts = {
-        get: store.query<{ value: string; expires_at: number | null }, [string, string]>(
-          "SELECT value, expires_at FROM kv WHERE ns = ? AND key = ?",
+        get: store.query<{ value: string; expires_at: number | null; binary: number }, [string, string]>(
+          "SELECT value, expires_at, binary FROM kv WHERE ns = ? AND key = ?",
         ),
         put: store.query(
-          "INSERT INTO kv (ns, key, value, expires_at) VALUES (?1, ?2, ?3, ?4) " +
-            "ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4",
+          "INSERT INTO kv (ns, key, value, expires_at, binary) VALUES (?1, ?2, ?3, ?4, ?5) " +
+            "ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5",
         ),
         del: store.query("DELETE FROM kv WHERE ns = ? AND key = ?"),
         // #58 — an expired row is filtered out here; a periodic sweep to hard-delete
@@ -1119,12 +1122,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
    * end the process. 32 MiB by default, and the same variable the embedded
    * transport reads so both backends agree.
    */
-  async function readCapped(response: Response, host: string): Promise<string> {
+  async function readCapped(response: Response, host: string): Promise<Buffer> {
     const cap = Number(process.env.SB_FETCH_MAX_BYTES) || 32 * 1024 * 1024;
     const declared = Number(response.headers.get("content-length") || 0);
     if (declared > cap) throw new Error(`response exceeds SB_FETCH_MAX_BYTES from ${host}`);
     const reader = response.body?.getReader();
-    if (!reader) return "";
+    if (!reader) return Buffer.alloc(0);
     const parts: Uint8Array[] = [];
     let total = 0;
     for (;;) {
@@ -1137,10 +1140,27 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       }
       parts.push(value);
     }
-    return new TextDecoder().decode(Buffer.concat(parts));
+    return Buffer.concat(parts);
   }
 
-  async function proxyFetch(msg: Frame): Promise<Frame> {
+  /**
+   * #232 — an upstream response as metadata plus its raw bytes. The v0 frame
+   * carries the body as UTF-8 text inside the JSON (what already-deployed
+   * sprouts expect), which turns a binary body into U+FFFD soup; the v1 frame
+   * sends the bytes untouched in its binary section instead.
+   */
+  type Upstream = { reply: Frame; bytes: Buffer };
+  const asTextFrame = (u: Upstream) => ({ ...u.reply, body: new TextDecoder().decode(u.bytes) });
+
+  /** #232 — a binary request body arrives one char per byte; send those bytes, not their UTF-8. */
+  const upstreamBody = (msg: Frame): string | Uint8Array<ArrayBuffer> | undefined =>
+    msg.body == null
+      ? undefined
+      : msg.bodyBinary === true
+        ? new Uint8Array(Buffer.from(str(msg.body), "latin1"))
+        : str(msg.body);
+
+  async function proxyFetch(msg: Frame): Promise<Upstream> {
     let url: URL;
     try {
       url = new URL(str(msg.url));
@@ -1160,12 +1180,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     const res = await doFetch(url, {
       method,
       headers,
-      body: msg.body == null || method === "GET" || method === "HEAD" ? undefined : str(msg.body),
+      body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
       redirect: "manual",
     });
     const outHeaders: Array<[string, string]> = [];
     res.headers.forEach((v, k) => outHeaders.push([k, v]));
-    return { ok: true, status: res.status, headers: outHeaders, body: await readCapped(res, url.host) };
+    return { reply: { ok: true, status: res.status, headers: outHeaders }, bytes: await readCapped(res, url.host) };
   }
 
   /**
@@ -1176,7 +1196,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
    * minus the trip through the network (and therefore minus TLS and the
    * default-deny egress rules, which is why this is not `fetch`).
    */
-  async function serviceFetch(msg: Frame): Promise<Frame> {
+  async function serviceFetch(msg: Frame): Promise<Upstream> {
     const binding = str(msg.binding);
     const declared = bindings.services.find((entry) => entry.binding === binding);
     if (!declared) throw new Error(`service not bound: ${binding}`);
@@ -1206,12 +1226,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     const res = await doFetch(target, {
       method,
       headers,
-      body: msg.body == null || method === "GET" || method === "HEAD" ? undefined : str(msg.body),
+      body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
       redirect: "manual",
     });
     const outHeaders: Array<[string, string]> = [];
     res.headers.forEach((v, k) => outHeaders.push([k, v]));
-    return { ok: true, status: res.status, headers: outHeaders, body: await readCapped(res, host) };
+    return { reply: { ok: true, status: res.status, headers: outHeaders }, bytes: await readCapped(res, host) };
   }
 
   async function dispatch(msg: Frame): Promise<Frame> {
@@ -1227,11 +1247,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
           stmts.del.run(part, str(msg.key));
           return { ok: true, found: false, value: null };
         }
-        return { ok: true, found: true, value: row.value };
+        return { ok: true, found: true, value: row.value, binary: row.binary === 1 };
       }
       case "kv.put": {
         const { store, part } = storeFor("kv", requireKv(msg.ns));
-        kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg));
+        kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg), msg.binary === true ? 1 : 0);
         return { ok: true };
       }
       case "kv.delete": {
@@ -1299,10 +1319,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
         return { ok: true, value: secrets[name] };
       }
       case "fetch":
-        return proxyFetch(msg);
+        return asTextFrame(await proxyFetch(msg));
 
       case "service.fetch":
-        return serviceFetch(msg);
+        return asTextFrame(await serviceFetch(msg));
 
       case "d1.query": {
         const conn = d1(requireD1(msg.db));
@@ -1695,6 +1715,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       };
     }
 
+    // #232 — a v1 fetch gets the upstream body as raw bytes, not decoded text.
+    if (msg.op === "fetch") return proxyFetch(msg);
+    if (msg.op === "service.fetch") return serviceFetch(msg);
+
     if (msg.op === "assets.get") {
       if (!bindings.assets) throw new Error("assets not bound");
       const reqPath = str(msg.path) || "/";