@sproutboat/wire 0.9.2 → 0.10.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @sproutboat/wire
 
+## 0.10.0
+
+### Minor Changes
+
+- 276fd90: New `doDb` option and `--do-db` flag: Durable Object storage and alarms can live in their own SQLite file instead of the per-deployment database (baronunread/sproutboat#207). The platform points it at a per-project file so an object's state survives redeploys, as on Workers. Without it, behaviour is unchanged.
+
+## 0.9.3
+
+### Patch Changes
+
+- 5dba469: Binary data survives KV and outbound `fetch()` bodies (baronunread/sproutboat#233, and the request half of #232).
+  - **KV:** `put()` of an `ArrayBuffer` or any typed array stores its bytes, flagged binary, instead of `String(view)` (`"0,1,2,..."`). `get(key, type | { type })` supports `"text"`, `"json"` and `"arrayBuffer"` as in Workers: a binary value read as text decodes as UTF-8, and a text value read as `arrayBuffer` is its UTF-8 encoding. `"stream"` throws a clear error. Existing stores gain the `binary` column on first use.
+  - **`fetch()` and service-binding request bodies:** an `ArrayBuffer` or view is sent as its exact bytes in both transports, instead of `"0,1,2,..."`. The standalone HTTP client also stopped truncating bodies at the first NUL byte.
+
 ## 0.9.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/wire",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/broker.test.ts

@@ -39,12 +39,25 @@ test("ping round-trips", async () => {
   expect(await b.dispatch({ op: "ping", msg: "hi" })).toMatchObject({ ok: true, op: "pong", echo: "hi" });
 });
 
+test("#233: a KV value put as bytes is flagged binary and survives an existing store", async () => {
+  const b = make({ bindings: { kv: ["CACHE"] } });
+  // One char per byte, as the runtime sends a put of an ArrayBuffer/view.
+  const bytes = Array.from({ length: 256 }, (_, i) => String.fromCharCode(i)).join("");
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "bin", value: bytes, binary: true });
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "txt", value: "café" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "bin" })).toEqual({ ok: true, found: true, value: bytes, binary: true });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "txt" })).toMatchObject({ value: "café", binary: false });
+  // Overwriting a binary value with text clears the flag.
+  await b.dispatch({ op: "kv.put", ns: "CACHE", key: "bin", value: "now text" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "bin" })).toMatchObject({ value: "now text", binary: false });
+});
+
 test("KV put / get / list / delete, scoped to a bound namespace", async () => {
   const b = make({ bindings: { kv: ["CACHE"] } });
   expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: false, value: null });
   await b.dispatch({ op: "kv.put", ns: "CACHE", key: "k", value: "v" });
   await b.dispatch({ op: "kv.put", ns: "CACHE", key: "k2", value: "v2" });
-  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: true, value: "v" });
+  expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toEqual({ ok: true, found: true, value: "v", binary: false });
   expect(await b.dispatch({ op: "kv.list", ns: "CACHE", prefix: "k" })).toEqual({ ok: true, keys: ["k", "k2"] });
   await b.dispatch({ op: "kv.delete", ns: "CACHE", key: "k" });
   expect(await b.dispatch({ op: "kv.get", ns: "CACHE", key: "k" })).toMatchObject({ found: false });
@@ -154,6 +167,21 @@ test("fetch is gated by the exact-host allowlist", async () => {
   expect(calls).toEqual(["https://api.example.com/x"]);
 });
 
+test("#232: a binary fetch request body is sent as its bytes, a text body as UTF-8", async () => {
+  const seen: Uint8Array[] = [];
+  const fetchImpl: FetchLike = async (_url, init) => {
+    seen.push(new Uint8Array(await new Response(init?.body).arrayBuffer()));
+    return new Response("ok");
+  };
+  const b = make({ bindings: { outbound: ["api.example.com"] }, fetchImpl });
+  // The runtime sends a binary body one char per byte, flagged bodyBinary.
+  const raw = Array.from({ length: 256 }, (_, i) => String.fromCharCode(i)).join("");
+  await b.dispatch({ op: "fetch", url: "https://api.example.com/up", method: "POST", body: raw, bodyBinary: true });
+  await b.dispatch({ op: "fetch", url: "https://api.example.com/up", method: "POST", body: "café" });
+  expect(Array.from(seen[0])).toEqual(Array.from({ length: 256 }, (_, i) => i));
+  expect(Array.from(seen[1])).toEqual([99, 97, 102, 195, 169]);
+});
+
 test("#232: a v1 fetch returns the upstream body as raw bytes; v0 keeps decoded text", async () => {
   const binary = Uint8Array.from({ length: 256 }, (_, i) => i);
   const fetchImpl: FetchLike = async (url) =>
@@ -678,6 +706,40 @@ test("cronMatches: fields, steps, ranges, lists (UTC)", () => {
 // #74 — a binding wired to an account-level resource stores in its own file
 // under resourceDir, keyed by id, so the data outlives a "redeploy" (a new
 // broker with a fresh per-deployment db but the same resourceDir).
+test("#207: Durable Object storage and alarms in doDb outlive a redeploy", async () => {
+  const root = mkdtempSync(join(tmpdir(), "sb-broker-do-"));
+  try {
+    mkdirSync(join(root, "dep-a"), { recursive: true });
+    mkdirSync(join(root, "dep-b"), { recursive: true });
+    const doDb = join(root, "do_0123456789abcdef01234567.sqlite");
+    const bindings = { do: [{ binding: "COUNTER", className: "Counter" }] };
+
+    const first = createBroker({ db: join(root, "dep-a", "state.sqlite"), doDb, bindings });
+    await first.dispatch({ op: "do.storage.put", cls: "Counter", id: "global", key: "n", value: "41" });
+    await first.dispatch({ op: "do.alarm.set", cls: "Counter", id: "global", at: 1234 });
+    first.close();
+
+    // A redeploy: new per-deployment state.sqlite, same per-project doDb.
+    const second = createBroker({ db: join(root, "dep-b", "state.sqlite"), doDb, bindings });
+    expect(await second.dispatch({ op: "do.storage.get", cls: "Counter", id: "global", key: "n" })).toEqual({
+      ok: true,
+      found: true,
+      value: "41",
+    });
+    expect(await second.dispatch({ op: "do.alarm.get", cls: "Counter", id: "global" })).toMatchObject({ at: 1234 });
+    second.close();
+
+    // Without doDb the state stays per deployment, as before.
+    const third = createBroker({ db: join(root, "dep-a", "state.sqlite"), bindings });
+    expect(await third.dispatch({ op: "do.storage.get", cls: "Counter", id: "global", key: "n" })).toMatchObject({
+      found: false,
+    });
+    third.close();
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("resource-backed KV persists across brokers and is shared by id", async () => {
   const root = mkdtempSync(join(tmpdir(), "sb-broker-res-"));
   try {
@@ -705,6 +767,7 @@ test("resource-backed KV persists across brokers and is shared by id", async ()
     expect(await second.dispatch({ op: "kv.get", ns: "SHORTENER", key: "a" })).toEqual({
       ok: true,
       found: true,
+      binary: false,
       value: "1",
     });
     second.close();

package/src/broker.ts

@@ -85,6 +85,12 @@ export type BrokerOptions = {
    * data outlives a redeploy.
    */
   resourceDir?: string;
+  /**
+   * #207 — SQLite file for Durable Object storage and alarms. The supervisor
+   * points this at a per-project path, so an object's state outlives a
+   * redeploy as it does on Workers. Defaults to the per-broker `db`.
+   */
+  doDb?: string;
   /** Account whose R2 resources this broker may mutate. */
   ownerId?: string;
   /** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
@@ -269,6 +275,9 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     );
     const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
     if (!kvColumns.some((column) => column.name === "expires_at")) conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
+    // #233 — 1 when the value was put as bytes (stored one char per byte), so get()
+    // can hand back exact bytes or decoded text depending on the requested type.
+    if (!kvColumns.some((column) => column.name === "binary")) conn.exec("ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0");
     // #56 — no `body` column: an object's bytes live in their own file under
     // `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
     // Doubling every object through SQLite's own page cache on top of the
@@ -309,9 +318,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   };
 
   const db = openStore(dbPath);
-  // Durable Object storage and Analytics Engine rows stay in the per-broker db —
-  // neither is an account-level resource (#74).
-  db.exec(
+  // #207 — Durable Object state lives in its own file when the supervisor gives
+  // one (per project, stable across deploys); otherwise in the per-broker db.
+  const doDb = opts.doDb ? new Database(opts.doDb, { create: true }) : db;
+  if (doDb !== db) {
+    doDb.exec("PRAGMA journal_mode = WAL");
+    doDb.exec("PRAGMA synchronous = NORMAL");
+  }
+  doDb.exec(
     "CREATE TABLE IF NOT EXISTS do_storage (cls TEXT NOT NULL, id TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, " +
       "PRIMARY KEY (cls, id, key))",
   );
@@ -321,12 +335,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   );
   // #125 — at most one pending alarm per object, which is the Workers rule: a
   // later setAlarm replaces the earlier one rather than queueing beside it.
-  db.exec(
+  doDb.exec(
     "CREATE TABLE IF NOT EXISTS do_alarm (cls TEXT NOT NULL, id TEXT NOT NULL, at INTEGER NOT NULL, " +
       "attempts INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (cls, id))",
   );
   // Alarm polling reads the earliest due objects across classes.
-  db.exec("CREATE INDEX IF NOT EXISTS do_alarm_due ON do_alarm (at)");
+  doDb.exec("CREATE INDEX IF NOT EXISTS do_alarm_due ON do_alarm (at)");
   // #69 — fixed-window rate-limiter counters, per binding name + key.
   db.exec(
     "CREATE TABLE IF NOT EXISTS ratelimit (name TEXT NOT NULL, key TEXT NOT NULL, window_start INTEGER NOT NULL, " +
@@ -481,8 +495,8 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   };
 
   type KvStmts = {
-    get: Statement<{ value: string; expires_at: number | null }, [string, string]>;
-    put: Statement<unknown, [string, string, string, number | null]>;
+    get: Statement<{ value: string; expires_at: number | null; binary: number }, [string, string]>;
+    put: Statement<unknown, [string, string, string, number | null, number]>;
     del: Statement<unknown, [string, string]>;
     list: Statement<{ key: string }, [string, string, number]>;
   };
@@ -491,12 +505,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     let stmts = kvStmtCache.get(store);
     if (!stmts) {
       stmts = {
-        get: store.query<{ value: string; expires_at: number | null }, [string, string]>(
-          "SELECT value, expires_at FROM kv WHERE ns = ? AND key = ?",
+        get: store.query<{ value: string; expires_at: number | null; binary: number }, [string, string]>(
+          "SELECT value, expires_at, binary FROM kv WHERE ns = ? AND key = ?",
         ),
         put: store.query(
-          "INSERT INTO kv (ns, key, value, expires_at) VALUES (?1, ?2, ?3, ?4) " +
-            "ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4",
+          "INSERT INTO kv (ns, key, value, expires_at, binary) VALUES (?1, ?2, ?3, ?4, ?5) " +
+            "ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5",
         ),
         del: store.query("DELETE FROM kv WHERE ns = ? AND key = ?"),
         // #58 — an expired row is filtered out here; a periodic sweep to hard-delete
@@ -1149,6 +1163,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   type Upstream = { reply: Frame; bytes: Buffer };
   const asTextFrame = (u: Upstream) => ({ ...u.reply, body: new TextDecoder().decode(u.bytes) });
 
+  /** #232 — a binary request body arrives one char per byte; send those bytes, not their UTF-8. */
+  const upstreamBody = (msg: Frame): string | Uint8Array<ArrayBuffer> | undefined =>
+    msg.body == null
+      ? undefined
+      : msg.bodyBinary === true
+        ? new Uint8Array(Buffer.from(str(msg.body), "latin1"))
+        : str(msg.body);
+
   async function proxyFetch(msg: Frame): Promise<Upstream> {
     let url: URL;
     try {
@@ -1169,7 +1191,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     const res = await doFetch(url, {
       method,
       headers,
-      body: msg.body == null || method === "GET" || method === "HEAD" ? undefined : str(msg.body),
+      body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
       redirect: "manual",
     });
     const outHeaders: Array<[string, string]> = [];
@@ -1215,7 +1237,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     const res = await doFetch(target, {
       method,
       headers,
-      body: msg.body == null || method === "GET" || method === "HEAD" ? undefined : str(msg.body),
+      body: method === "GET" || method === "HEAD" ? undefined : upstreamBody(msg),
       redirect: "manual",
     });
     const outHeaders: Array<[string, string]> = [];
@@ -1236,11 +1258,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
           stmts.del.run(part, str(msg.key));
           return { ok: true, found: false, value: null };
         }
-        return { ok: true, found: true, value: row.value };
+        return { ok: true, found: true, value: row.value, binary: row.binary === 1 };
       }
       case "kv.put": {
         const { store, part } = storeFor("kv", requireKv(msg.ns));
-        kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg));
+        kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg), msg.binary === true ? 1 : 0);
         return { ok: true };
       }
       case "kv.delete": {
@@ -1553,7 +1575,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
 
       case "do.storage.get": {
         const cls = requireDoClass(msg.cls);
-        const row = db
+        const row = doDb
           .query<{ value: string }, [string, string, string]>(
             "SELECT value FROM do_storage WHERE cls = ? AND id = ? AND key = ?",
           )
@@ -1561,24 +1583,24 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
         return row ? { ok: true, found: true, value: row.value } : { ok: true, found: false };
       }
       case "do.storage.put":
-        db.query(
+        doDb.query(
           "INSERT INTO do_storage (cls, id, key, value) VALUES (?1,?2,?3,?4) " +
             "ON CONFLICT (cls, id, key) DO UPDATE SET value = ?4",
         ).run(requireDoClass(msg.cls), str(msg.id), str(msg.key), str(msg.value));
         return { ok: true };
       case "do.storage.delete": {
-        const r = db
+        const r = doDb
           .query("DELETE FROM do_storage WHERE cls = ? AND id = ? AND key = ?")
           .run(requireDoClass(msg.cls), str(msg.id), str(msg.key));
         return { ok: true, deleted: r.changes > 0 };
       }
       case "do.storage.delete_all":
-        db.query("DELETE FROM do_storage WHERE cls = ? AND id = ?").run(requireDoClass(msg.cls), str(msg.id));
+        doDb.query("DELETE FROM do_storage WHERE cls = ? AND id = ?").run(requireDoClass(msg.cls), str(msg.id));
         return { ok: true };
       case "do.storage.list": {
         const cls = requireDoClass(msg.cls);
         const limit = Math.min(Math.max(Number(msg.limit) || 1000, 1), 10000);
-        const rows = db
+        const rows = doDb
           .query<{ key: string; value: string }, [string, string, string, number]>(
             "SELECT key, value FROM do_storage WHERE cls = ? AND id = ? AND key LIKE ? || '%' ORDER BY key LIMIT ?",
           )
@@ -1589,19 +1611,19 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       // #125 — alarms. `at` is epoch ms; setting one replaces any pending alarm
       // for that object, and attempts resets because this is a fresh schedule.
       case "do.alarm.set":
-        db.query(
+        doDb.query(
           "INSERT INTO do_alarm (cls, id, at, attempts) VALUES (?1,?2,?3,0) " +
             "ON CONFLICT (cls, id) DO UPDATE SET at = ?3, attempts = 0",
         ).run(requireDoClass(msg.cls), str(msg.id), Math.trunc(Number(msg.at) || 0));
         return { ok: true };
       case "do.alarm.get": {
-        const row = db
+        const row = doDb
           .query<{ at: number }, [string, string]>("SELECT at FROM do_alarm WHERE cls = ? AND id = ?")
           .get(requireDoClass(msg.cls), str(msg.id));
         return { ok: true, at: row ? row.at : null };
       }
       case "do.alarm.delete": {
-        const r = db.query("DELETE FROM do_alarm WHERE cls = ? AND id = ?").run(requireDoClass(msg.cls), str(msg.id));
+        const r = doDb.query("DELETE FROM do_alarm WHERE cls = ? AND id = ?").run(requireDoClass(msg.cls), str(msg.id));
         return { ok: true, deleted: r.changes > 0 };
       }
 
@@ -1919,14 +1941,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
   function fireAlarmsOnce(): void {
     if (!opts.sproutUrl || !dispatchEnabled() || bindings.do.length === 0) return;
     const now = Date.now();
-    const due = db
+    const due = doDb
       .query<{ cls: string; id: string; at: number; attempts: number }, [number]>(
         "SELECT cls, id, at, attempts FROM do_alarm WHERE at <= ? ORDER BY at LIMIT 10",
       )
       .all(now);
     if (due.length === 0) return;
-    const claim = db.query("DELETE FROM do_alarm WHERE cls = ? AND id = ?");
-    const rearm = db.query(
+    const claim = doDb.query("DELETE FROM do_alarm WHERE cls = ? AND id = ?");
+    const rearm = doDb.query(
       "INSERT INTO do_alarm (cls, id, at, attempts) VALUES (?1,?2,?3,?4) ON CONFLICT (cls, id) DO NOTHING",
     );
     for (const row of due) {
@@ -1969,6 +1991,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
       for (const t of timers) clearInterval(t);
       for (const conn of d1Conns.values()) conn.close();
       for (const conn of resourceDbs.values()) conn.close();
+      if (doDb !== db) doDb.close();
       db.close();
     },
   };
@@ -2075,6 +2098,7 @@ if (import.meta.main) {
       db: { type: "string" },
       "data-dir": { type: "string" },
       "resource-dir": { type: "string" },
+      "do-db": { type: "string" },
       "owner-id": { type: "string" },
       "r2-quota-bytes": { type: "string" },
       "r2-resource-ids": { type: "string" },
@@ -2127,6 +2151,7 @@ if (import.meta.main) {
     db: values.db,
     dataDir: values["data-dir"],
     resourceDir: values["resource-dir"],
+    doDb: values["do-db"],
     ownerId: values["owner-id"],
     r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
     r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,