@sproutboat/wire 0.5.0 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @sproutboat/wire
 
+## 0.6.0
+
+### Minor Changes
+
+- 6998a15: R2 object bytes no longer live inside SQLite, on either transport (baronunread/sproutboat#56). Every `put()` used to bind the whole object into a `body` column, and every `get()` read it back out — doubling the object through SQLite's own page cache/WAL machinery on top of the buffer it already arrived in, which was the real driver of #56's "large uploads make memory skyrocket" complaint (the JSON-escaping half of that was already fixed in #63).
+  
+  Objects now live in their own file under `r2-blobs/`, named by a hash of bucket+key (so unicode keys and directory traversal are non-issues):
+  
+  - **Embedded (standalone):** `<data-dir>/r2-blobs/` — `sb_r2_put_c`/`sb_r2_get_c` write/read the file directly via C `fopen`/`fwrite`/`fread`, no SQLite blob column at all. `sb_r2_delete_blob` removes the file on delete.
+  - **Broker:** colocated with whichever store file already holds that bucket's metadata — the main `db` for a bare-string binding, the resource's own file under `resourceDir` for an account-level one (#74) — so blobs persist across a redeploy exactly when that metadata does. `:memory:` stores (tests, local dev) fall back to an in-memory map.
+  
+  No migration: this is a schema change for new writes going forward, not a converter for objects already stored inline from before.
+  
+  Verified end-to-end against both transports (kitchen-sink's full conformance suite, 24/24 standalone + 27/27 broker) and with new unit tests for file-backed persistence, the `:memory:` fallback, and a real binary round-trip over the v1 wire protocol.
+  
+  **Known limitation found while verifying this, not fixed here:** a binary `put()` body (bytes ≥ 0x80) is corrupted on the way *in*, before it ever reaches storage — `porf_native_fetch_read_value` (the function every inline-C string read in the runtime goes through) always UTF-8-encodes a Latin-1-range string on read, which is correct for genuine text but wrong for opaque bytes coming off a raw HTTP body. This predates this change (the old SQLite-blob path had the identical corruption) and is not something the storage backend can fix — it needs an input-side counterpart to `#176`'s `x-sb-raw-body` marker, or real `ArrayBuffer`/`Uint8Array` body support (the upstream ask already filed as Draft G in `patches/UPSTREAM.md`).
+
 ## 0.5.0
 
 ### Minor Changes

package/package.json

@@ -1,8 +1,13 @@
 {
   "name": "@sproutboat/wire",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "license": "MIT",
   "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/baronunread/sproutboat-packages.git",
+    "directory": "packages/wire"
+  },
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "dependencies": {

package/src/broker.test.ts

@@ -1,6 +1,6 @@
 import { afterEach, expect, test } from "bun:test";
 import { Database } from "bun:sqlite";
-import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
@@ -332,6 +332,87 @@ test("resource-backed KV persists across brokers and is shared by id", async ()
   }
 });
 
+// #56 — object bytes live in their own file next to the store's own SQLite
+// file, not inside it, so a resource-bound bucket's blobs persist across a
+// redeploy exactly like its metadata does, and a bare-string binding's blobs
+// live beside its per-deployment db.
+test("R2 blobs are file-backed and persist across brokers for a resource-bound bucket", async () => {
+  const root = mkdtempSync(join(tmpdir(), "sb-broker-r2-res-"));
+  try {
+    const resourceDir = join(root, "resources");
+    const bucketId = "r2_0123456789abcdef01234567";
+    mkdirSync(join(root, "dep-a"), { recursive: true });
+    mkdirSync(join(root, "dep-b"), { recursive: true });
+
+    const first = createBroker({
+      db: join(root, "dep-a", "state.sqlite"),
+      resourceDir,
+      bindings: { r2: ["UPLOADS"], resources: { UPLOADS: { kind: "r2", id: bucketId } } },
+    });
+    await first.dispatch({ op: "r2.put", bucket: "UPLOADS", key: "a.txt", body: "hello" });
+    first.close();
+    // metadata in the per-resource file, bytes in their own file beside it —
+    // never inline in the sqlite row.
+    const dbFile = join(resourceDir, `${bucketId}.sqlite`);
+    expect(existsSync(dbFile)).toBe(true);
+    expect(readFileSync(dbFile, "utf8")).not.toContain("hello");
+    expect(existsSync(join(resourceDir, "r2-blobs"))).toBe(true);
+
+    const second = createBroker({
+      db: join(root, "dep-b", "state.sqlite"),
+      resourceDir,
+      bindings: { r2: ["FILES"], resources: { FILES: { kind: "r2", id: bucketId } } },
+    });
+    expect(await second.dispatch({ op: "r2.get", bucket: "FILES", key: "a.txt" })).toMatchObject({
+      ok: true,
+      found: true,
+      body: "hello",
+    });
+    second.close();
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("a bare-string R2 binding stores blobs beside the per-deployment db, not resourceDir", async () => {
+  const root = mkdtempSync(join(tmpdir(), "sb-broker-r2-bare-"));
+  try {
+    const resourceDir = join(root, "resources");
+    mkdirSync(join(root, "dep"), { recursive: true });
+    const b = createBroker({ db: join(root, "dep", "state.sqlite"), resourceDir, bindings: { r2: ["UPLOADS"] } });
+    const put = await b.dispatch({ op: "r2.put", bucket: "UPLOADS", key: "k", body: "bytes" });
+    b.close();
+    expect(existsSync(resourceDir)).toBe(false);
+    expect(existsSync(join(root, "dep", "r2-blobs"))).toBe(true);
+    expect(obj(put.object)).toMatchObject({ size: 5 });
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+// Binary round trip against a real file-backed store (not `:memory:`), the
+// v1 path a live sprout actually uses (#63).
+test("R2 v1 binary put/get round-trips through a file-backed blob", async () => {
+  const root = mkdtempSync(join(tmpdir(), "sb-broker-r2-v1-"));
+  try {
+    const b = createBroker({ db: join(root, "state.sqlite"), bindings: { r2: ["UPLOADS"] }, token: "tok" });
+    const server = listen(b, "127.0.0.1", 0);
+    try {
+      const body = new Uint8Array([0, 1, 2, 0x80, 0xff, 254, 253]);
+      const put = await v1(server, { v: 1, token: "tok", op: "r2.put", bucket: "UPLOADS", key: "k" }, body);
+      expect(obj(put.json.object)).toMatchObject({ size: body.length });
+      expect(existsSync(join(root, "r2-blobs"))).toBe(true);
+      const got = await v1(server, { v: 1, token: "tok", op: "r2.get", bucket: "UPLOADS", key: "k" });
+      expect(Buffer.from(got.bytes).equals(Buffer.from(body))).toBe(true);
+    } finally {
+      server.stop();
+      b.close();
+    }
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("a bare-string binding still uses the per-broker db, not resourceDir", async () => {
   const root = mkdtempSync(join(tmpdir(), "sb-broker-bare-"));
   try {

package/src/broker.ts

@@ -19,7 +19,7 @@
  */
 import { Database, type Statement } from "bun:sqlite";
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, normalize, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
@@ -110,8 +110,6 @@ type D1Result = {
 };
 type R2Row = {
   key: string;
-  /** TEXT from a v0 write, BLOB from a v1 one; sqlite hands each back as-is. */
-  body: string | Uint8Array;
   size: number;
   etag: string;
   uploaded: string;
@@ -234,8 +232,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     conn.exec(
       "CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (ns, key))",
     );
+    // #56 — no `body` column: an object's bytes live in their own file under
+    // `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
+    // Doubling every object through SQLite's own page cache on top of the
+    // buffer it already arrived in was the real cost driving #56's "large
+    // uploads make memory skyrocket" complaint, not just the JSON-escaping
+    // #63 already fixed.
     conn.exec(
-      "CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, body TEXT NOT NULL, size INTEGER NOT NULL, " +
+      "CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
         "etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
         "PRIMARY KEY (bucket, key))",
     );
@@ -414,15 +418,48 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
     return { path: dest, bytes: statSync(dest).size };
   };
 
-  /** An object body as bytes, whichever storage class it came back in. */
-  const r2Bytes = (body: string | Uint8Array): Uint8Array =>
-    body instanceof Uint8Array ? body : new TextEncoder().encode(body);
-
-  /** An object body as text, for the v0 reply shape that carries it in JSON.
-   *  A binary object stored by a v1 client cannot survive this, which is why
-   *  v1 exists — but returning a decoded string beats returning `{"0":...}`. */
-  const r2Text = (body: string | Uint8Array): string =>
-    body instanceof Uint8Array ? new TextDecoder().decode(body) : body;
+  /**
+   * #56 — an object's bytes never enter SQLite; they live in their own file,
+   * named by a hash of bucket+key (so unicode keys and directory traversal
+   * are both non-issues), colocated with whichever store file holds that
+   * bucket's metadata — the main `db` for a bare-string binding, the
+   * resource's own file under `resourceDir` for an account-level one (#74) —
+   * so blobs persist across a redeploy exactly when that metadata does.
+   * `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
+   */
+  const r2MemBlobs = new Map<string, Uint8Array>();
+  const r2BlobId = (bucketPart: string, key: string): string =>
+    createHash("sha256").update(`${bucketPart}\0${key}`).digest("hex");
+  const r2BlobDir = (store: Database): string | null => {
+    const file = store.filename;
+    return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
+  };
+  const r2BlobPath = (store: Database, bucketPart: string, key: string): string | null => {
+    const dir = r2BlobDir(store);
+    return dir ? join(dir, `${r2BlobId(bucketPart, key)}.blob`) : null;
+  };
+  const r2WriteBlob = (store: Database, bucketPart: string, key: string, bytes: Uint8Array): void => {
+    const path = r2BlobPath(store, bucketPart, key);
+    if (!path) {
+      r2MemBlobs.set(`${bucketPart}\0${key}`, bytes);
+      return;
+    }
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, bytes);
+  };
+  const r2ReadBlob = (store: Database, bucketPart: string, key: string): Uint8Array | null => {
+    const path = r2BlobPath(store, bucketPart, key);
+    if (!path) return r2MemBlobs.get(`${bucketPart}\0${key}`) ?? null;
+    return existsSync(path) ? readFileSync(path) : null;
+  };
+  const r2DeleteBlob = (store: Database, bucketPart: string, key: string): void => {
+    const path = r2BlobPath(store, bucketPart, key);
+    if (!path) {
+      r2MemBlobs.delete(`${bucketPart}\0${key}`);
+      return;
+    }
+    if (existsSync(path)) rmSync(path);
+  };
 
   const r2Row = (r: R2Row) => ({
     key: r.key,
@@ -617,40 +654,42 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
 
       case "r2.put": {
         const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
-        const body = str(msg.body);
-        const etag = createHash("sha256").update(body).digest("hex");
+        const key = str(msg.key);
+        const bytes = new TextEncoder().encode(str(msg.body));
+        const etag = createHash("sha256").update(bytes).digest("hex");
+        const uploaded = new Date().toISOString();
+        r2WriteBlob(store, bucket, key, bytes);
         store
           .query(
-            "INSERT INTO r2 (bucket, key, body, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
-              "ON CONFLICT (bucket, key) DO UPDATE SET body=?3, size=?4, etag=?5, uploaded=?6, http_json=?7, custom_json=?8",
+            "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7) " +
+              "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, custom_json=?7",
           )
           .run(
             bucket,
-            str(msg.key),
-            body,
-            Buffer.byteLength(body),
+            key,
+            bytes.byteLength,
             etag,
-            new Date().toISOString(),
+            uploaded,
             JSON.stringify(msg.httpMetadata ?? {}),
             JSON.stringify(msg.customMetadata ?? {}),
           );
-        return {
-          ok: true,
-          object: { key: str(msg.key), size: Buffer.byteLength(body), etag, uploaded: new Date().toISOString() },
-        };
+        return { ok: true, object: { key, size: bytes.byteLength, etag, uploaded } };
       }
       case "r2.get":
       case "r2.head": {
         const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
-        const row = store
-          .query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
-          .get(bucket, str(msg.key));
+        const key = str(msg.key);
+        const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
         if (!row) return { ok: true, found: false };
-        return { ok: true, found: true, object: r2Row(row), body: msg.op === "r2.get" ? r2Text(row.body) : undefined };
+        const body =
+          msg.op === "r2.get" ? new TextDecoder().decode(r2ReadBlob(store, bucket, key) ?? new Uint8Array(0)) : undefined;
+        return { ok: true, found: true, object: r2Row(row), body };
       }
       case "r2.delete": {
         const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
-        store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, str(msg.key));
+        const key = str(msg.key);
+        store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
+        r2DeleteBlob(store, bucket, key);
         return { ok: true };
       }
       case "r2.list": {
@@ -832,38 +871,37 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
    */
   async function dispatchBinary(msg: Frame, binary: Uint8Array): Promise<{ reply: Frame; bytes?: Uint8Array }> {
     if (msg.op === "r2.put") {
-      // The body arrived as bytes; store it as a blob rather than a string, so
-      // a put costs one copy instead of an escape, a parse and a re-encode.
+      // The body arrived as bytes; write it straight to its own file rather
+      // than into SQLite (#56) — no escape, no parse, no re-encode, and no
+      // second full-object copy through the database's own page cache.
       const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
+      const key = str(msg.key);
       const etag = createHash("sha256").update(binary).digest("hex");
       const uploaded = new Date().toISOString();
+      r2WriteBlob(store, bucket, key, binary);
       store
         .query(
-          "INSERT INTO r2 (bucket, key, body, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
-            "ON CONFLICT (bucket, key) DO UPDATE SET body=?3, size=?4, etag=?5, uploaded=?6, http_json=?7, custom_json=?8",
+          "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7) " +
+            "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, custom_json=?7",
         )
         .run(
           bucket,
-          str(msg.key),
-          binary,
+          key,
           binary.byteLength,
           etag,
           uploaded,
           JSON.stringify(msg.httpMetadata ?? {}),
           JSON.stringify(msg.customMetadata ?? {}),
         );
-      return {
-        reply: { ok: true, object: { key: str(msg.key), size: binary.byteLength, etag, uploaded } },
-      };
+      return { reply: { ok: true, object: { key, size: binary.byteLength, etag, uploaded } } };
     }
 
     if (msg.op === "r2.get") {
       const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
-      const row = store
-        .query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
-        .get(bucket, str(msg.key));
+      const key = str(msg.key);
+      const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
       if (!row) return { reply: { ok: true, found: false } };
-      return { reply: { ok: true, found: true, object: r2Row(row) }, bytes: r2Bytes(row.body) };
+      return { reply: { ok: true, found: true, object: r2Row(row) }, bytes: r2ReadBlob(store, bucket, key) ?? new Uint8Array(0) };
     }
 
     if (msg.op === "assets.get") {