@sproutboat/runtime 0.13.0 → 0.13.1

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @sproutboat/runtime
 
+## 0.13.1
+
+### Patch Changes
+
+- ebee2ad: Non-ASCII text read back from bindings no longer comes back garbled (baronunread/sproutboat#189). D1 rows, KV values, cached bodies, R2 metadata and SQLite error messages returned "café" as "café" in both standalone builds and broker-backed deployments: the reply JSON is UTF-8, but it reached the handler one char per byte. Replies are now escaped to ASCII before parsing. Outbound `fetch()` bodies and BLOB columns keep their raw bytes.
+
 ## 0.13.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/runtime",
-  "version": "0.13.0",
+  "version": "0.13.1",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/native-fetch-prelude.js

@@ -1272,9 +1272,9 @@ function __sbMakeD1(dbName) {
       return { count: (String(sql).match(/;/g) || []).length, duration: 0 };
     },
     // Sproutboat extension (not in CF Workers D1): an online, integrity-checked
-    // snapshot of this database. Standalone only for now — the broker transport
-    // returns an error until hosted backup (#139) covers it. `name` lands under
-    // `<data-dir>/backups/`; omit it for a timestamped default. See #164.
+    // snapshot of this database. Both transports implement it: standalone writes
+    // under `<data-dir>/backups/`, the broker under the resource directory's
+    // `backups/`. Omit `name` for a timestamped default. See #164.
     backup(name) {
       const r = __sbRpc("d1.backup", { db: dbName, name: name == null ? "" : String(name) });
       return { path: r.path, bytes: r.bytes };

package/src/transport-broker.js

@@ -147,6 +147,66 @@ static int sb_broker_roundtrip(const char* req, size_t req_len, char** resp_out,
   return rc;
 }
 
+// baronunread/sproutboat#189 -- the broker writes its reply JSON as UTF-8,
+// but it reaches JS as a bytestring, one char per byte, so any non-ASCII text
+// (a D1 row, a KV value, a cached body) came back as Latin-1 mojibake. Non-ASCII
+// bytes only ever sit inside JSON strings, so rewriting each UTF-8 sequence as
+// \uXXXX gives the same JSON in pure ASCII, and JSON.parse rebuilds the real
+// strings. A byte that starts no valid sequence passes through as \u00XX.
+// Returns a malloc'd copy, or 0 when the reply is already ASCII.
+static char* sb_json_ascii(const char* s, size_t n, size_t* out_len) {
+  size_t high = 0;
+  for (size_t i = 0; i < n; i++) if ((unsigned char)s[i] >= 0x80) high++;
+  if (!high) return 0;
+  char* out = (char*)malloc(n + high * 12 + 1);
+  if (!out) return 0;
+  size_t o = 0;
+  for (size_t i = 0; i < n; i++) {
+    unsigned char c = (unsigned char)s[i];
+    if (c < 0x80) { out[o++] = (char)c; continue; }
+    unsigned int cp = 0, min = 0;
+    size_t len = 0;
+    if ((c & 0xe0) == 0xc0) { cp = c & 0x1f; len = 2; min = 0x80; }
+    else if ((c & 0xf0) == 0xe0) { cp = c & 0x0f; len = 3; min = 0x800; }
+    else if ((c & 0xf8) == 0xf0) { cp = c & 0x07; len = 4; min = 0x10000; }
+    int ok = len > 0 && i + len <= n;
+    for (size_t k = 1; ok && k < len; k++) {
+      unsigned char cc = (unsigned char)s[i + k];
+      if ((cc & 0xc0) != 0x80) ok = 0;
+      else cp = (cp << 6) | (cc & 0x3f);
+    }
+    if (ok && (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))) ok = 0;
+    if (!ok) { o += (size_t)sprintf(out + o, "\\u%04x", c); continue; }
+    if (cp >= 0x10000) {
+      cp -= 0x10000;
+      o += (size_t)sprintf(out + o, "\\u%04x\\u%04x", 0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff));
+    } else {
+      o += (size_t)sprintf(out + o, "\\u%04x", cp);
+    }
+    i += len - 1;
+  }
+  out[o] = 0;
+  *out_len = o;
+  return out;
+}
+
+// fetch and service.fetch are the exception: their body is the upstream's
+// bytes, which the prelude keeps one char per byte (x-sb-raw-body) and decodes
+// itself. __sbRpc always writes {"v":1,"id":N,"op":... first, so the op is a
+// fixed-position prefix, not a search through user data.
+static int sb_req_is_fetch(const char* req, size_t n) {
+  const char* head = "{\"v\":1,\"id\":";
+  size_t hl = strlen(head);
+  if (n < hl || memcmp(req, head, hl) != 0) return 0;
+  size_t i = hl;
+  while (i < n && req[i] >= '0' && req[i] <= '9') i++;
+  const char* a = ",\"op\":\"fetch\"";
+  const char* b = ",\"op\":\"service.fetch\"";
+  if (n - i >= strlen(a) && memcmp(req + i, a, strlen(a)) == 0) return 1;
+  if (n - i >= strlen(b) && memcmp(req + i, b, strlen(b)) == 0) return 1;
+  return 0;
+}
+
 // A v1 exchange: marker, json length, json, then the body bytes. The reply is
 // split the same way, its JSON returned and its bytes stashed for __sbTakeBin.
 static int sb_broker_roundtrip_v1(const char* json, size_t json_len, const char* body, size_t body_len,
@@ -215,8 +275,12 @@ function __sbCall(reqJson) {
     porf_native_fetch_read_value(reqJson, &__req, &__reqlen, &__reqowned);
     char* __resp = 0; size_t __resplen = 0;
     int __rc = sb_broker_roundtrip(__req, __reqlen, &__resp, &__resplen);
+    int __raw = sb_req_is_fetch(__req, __reqlen);
     if (__reqowned) free(__reqowned);
     if (__rc == 0) {
+      size_t __al = 0;
+      char* __ascii = __raw ? 0 : sb_json_ascii(__resp, __resplen, &__al);
+      if (__ascii) { free(__resp); __resp = __ascii; __resplen = __al; }
       res = porf_box((f64)porf_native_fetch_alloc_bytestring(__resp, __resplen), 195);
       free(__resp);
     } else {
@@ -258,6 +322,9 @@ function __sbCallBin(reqJson, body) {
     if (__jo) free(__jo);
     if (__bo) free(__bo);
     if (__rc == 0) {
+      size_t __al = 0;
+      char* __ascii = sb_json_ascii(__resp, __resplen, &__al);
+      if (__ascii) { free(__resp); __resp = __ascii; __resplen = __al; }
       res = porf_box((f64)porf_native_fetch_alloc_bytestring(__resp, __resplen), 195);
       free(__resp);
     } else {

package/src/transport-embedded.js

@@ -184,6 +184,53 @@ static void sb_putjson(sb_buf* b, const char* s, size_t n) {
   }
   sb_put(b, "\"", 1);
 }
+// baronunread/sproutboat#189 -- sb_putjson for bytes that are UTF-8 text
+// (SQLite TEXT, its error messages). The reply reaches JS as a bytestring, one
+// char per byte, so a raw multi-byte sequence came back as Latin-1 mojibake
+// ("café" as "café"). Escaping every code point >= 0x80 as \uXXXX keeps the
+// reply pure ASCII and lets JSON.parse rebuild the real string. A byte that
+// doesn't start a valid sequence passes through as \u00XX, the same rule as
+// the prelude's __sbFromUtf8. Raw byte producers (the HTTP client's body,
+// BLOB columns) keep sb_putjson.
+static void sb_putjson_text(sb_buf* b, const char* s, size_t n) {
+  sb_put(b, "\"", 1);
+  for (size_t i = 0; i < n; i++) {
+    unsigned char c = (unsigned char)s[i];
+    if (c < 0x80) {
+      if (c == '"' || c == '\\') { char e[2] = { '\\', (char)c }; sb_put(b, e, 2); }
+      else if (c == '\n') sb_put(b, "\\n", 2);
+      else if (c == '\r') sb_put(b, "\\r", 2);
+      else if (c == '\t') sb_put(b, "\\t", 2);
+      else if (c < 0x20) { char e[8]; int k = snprintf(e, 8, "\\u%04x", c); sb_put(b, e, (size_t)k); }
+      else sb_put(b, (const char*)&c, 1);
+      continue;
+    }
+    unsigned int cp = 0, min = 0;
+    size_t len = 0;
+    if ((c & 0xe0) == 0xc0) { cp = c & 0x1f; len = 2; min = 0x80; }
+    else if ((c & 0xf0) == 0xe0) { cp = c & 0x0f; len = 3; min = 0x800; }
+    else if ((c & 0xf8) == 0xf0) { cp = c & 0x07; len = 4; min = 0x10000; }
+    int ok = len > 0 && i + len <= n;
+    for (size_t k = 1; ok && k < len; k++) {
+      unsigned char cc = (unsigned char)s[i + k];
+      if ((cc & 0xc0) != 0x80) ok = 0;
+      else cp = (cp << 6) | (cc & 0x3f);
+    }
+    if (ok && (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))) ok = 0;
+    char e[16];
+    int k;
+    if (!ok) { k = snprintf(e, sizeof(e), "\\u%04x", c); sb_put(b, e, (size_t)k); continue; }
+    if (cp >= 0x10000) {
+      cp -= 0x10000;
+      k = snprintf(e, sizeof(e), "\\u%04x\\u%04x", 0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff));
+    } else {
+      k = snprintf(e, sizeof(e), "\\u%04x", cp);
+    }
+    sb_put(b, e, (size_t)k);
+    i += len - 1;
+  }
+  sb_put(b, "\"", 1);
+}
 
 // --- the narrow JSON reader: a flat array of null | number | string ----------
 // Only ever fed __sbSql's own params, which JS builds; anything unexpected
@@ -495,7 +542,7 @@ static char* sb_r2_put_c(const char* path, const char* bucket, const char* key,
     if (strcmp(previous, etag) != 0) unlink(blobpath);
     sb_puts(&out, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
-    sb_putjson(&out, m, strlen(m));
+    sb_putjson_text(&out, m, strlen(m));
     sb_puts(&out, "}");
     return out.p;
   }
@@ -817,7 +864,7 @@ static char* sb_sql_script(const char* path, const char* sql) {
   char* err = 0;
   if (sqlite3_exec(sb_dbs[idx], sql, 0, 0, &err) != 0) {
     sb_puts(&b, "{\"ok\":false,\"error\":");
-    sb_putjson(&b, err ? err : "exec failed", err ? strlen(err) : 11);
+    sb_putjson_text(&b, err ? err : "exec failed", err ? strlen(err) : 11);
     sb_puts(&b, "}");
     return b.p;
   }
@@ -837,7 +884,7 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
   if (!st) {
     sb_puts(&b, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
-    sb_putjson(&b, m, strlen(m));
+    sb_putjson_text(&b, m, strlen(m));
     sb_puts(&b, "}");
     return b.p;
   }
@@ -848,7 +895,7 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
   for (int i = 0; i < ncol; i++) {
     if (i) sb_puts(&b, ",");
     const char* name = sqlite3_column_name(st, i);
-    sb_putjson(&b, name ? name : "", name ? strlen(name) : 0);
+    sb_putjson_text(&b, name ? name : "", name ? strlen(name) : 0);
   }
   sb_puts(&b, "],\"rows\":[");
   int rc, first = 1;
@@ -868,7 +915,9 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
       }
       const unsigned char* txt = sqlite3_column_text(st, i);
       int n = sqlite3_column_bytes(st, i);
-      sb_putjson(&b, txt ? (const char*)txt : "", txt ? (size_t)n : 0);
+      // TEXT is UTF-8 and decodes (#189); a BLOB stays one char per byte.
+      if (t == 4) sb_putjson(&b, txt ? (const char*)txt : "", txt ? (size_t)n : 0);
+      else sb_putjson_text(&b, txt ? (const char*)txt : "", txt ? (size_t)n : 0);
     }
     sb_puts(&b, "]");
   }
@@ -883,7 +932,7 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
     sb_buf e = { 0, 0, 0 };
     sb_puts(&e, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
-    sb_putjson(&e, m, strlen(m));
+    sb_putjson_text(&e, m, strlen(m));
     sb_puts(&e, "}");
     return e.p;
   }
@@ -909,7 +958,7 @@ static char* sb_sql_backup(const char* src_path, const char* dest_path) {
   if (sqlite3_prepare_v2(db, "VACUUM INTO ?1", -1, &st, 0) != 0 || !st) {
     sb_puts(&b, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
-    sb_putjson(&b, m, strlen(m));
+    sb_putjson_text(&b, m, strlen(m));
     sb_puts(&b, "}");
     return b.p;
   }
@@ -919,7 +968,7 @@ static char* sb_sql_backup(const char* src_path, const char* dest_path) {
   if (rc != SB_SQLITE_DONE) {
     sb_puts(&b, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
-    sb_putjson(&b, m, strlen(m));
+    sb_putjson_text(&b, m, strlen(m));
     sb_puts(&b, "}");
     return b.p;
   }
@@ -1097,7 +1146,7 @@ static char* sb_error_json(const char* prefix, const char* detail) {
   sb_buf msg = { 0, 0, 0 };
   sb_puts(&msg, prefix);
   if (detail) { sb_puts(&msg, detail); }
-  sb_putjson(&out, msg.p ? msg.p : "", msg.len);
+  sb_putjson_text(&out, msg.p ? msg.p : "", msg.len);
   sb_puts(&out, "}");
   free(msg.p);
   return out.p;

package/test/native-utf8.ts

@@ -0,0 +1,114 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { ensurePorffor } from "../../toolchain/src/acquire";
+import { createBroker, listen } from "../../wire/src/broker";
+import { EMPTY_BINDINGS, preludePath, TRANSPORT_MARKER, transportPath, wrapNativeFetchHandler } from "../src/wrap";
+
+// baronunread/sproutboat#189: text read back through the broker came back as
+// Latin-1 mojibake ("café" as "café"), because the reply JSON is UTF-8 and the
+// sprout holds it one char per byte. Covers 2-, 3- and 4-byte sequences on the
+// two hot paths (KV, D1) and a non-ASCII SQLite error message.
+const text = "/café/日本/😀/x";
+
+const handler = `
+export default {
+  async fetch() {
+    await env.KV.put("k", ${JSON.stringify(text)});
+    const kv = await env.KV.get("k");
+    await env.DB.prepare("CREATE TABLE t (v TEXT)").run();
+    await env.DB.prepare("INSERT INTO t (v) VALUES (?)").bind(${JSON.stringify(text)}).run();
+    const d1 = (await env.DB.prepare("SELECT v FROM t").first()).v;
+    let error = "";
+    try { await env.DB.prepare("SELECT * FROM tabellà").first(); } catch (e) { error = String(e.message); }
+    return Response.json({ kv, kvLength: kv.length, d1, d1Length: d1.length, error });
+  },
+};
+`;
+
+function freePort(): number {
+  const listener = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
+  const port = listener.port;
+  listener.stop(true);
+  return port;
+}
+
+const workdir = await mkdtemp(join(tmpdir(), "sb-native-utf8-"));
+const bindings = { ...EMPTY_BINDINGS, kv: ["KV"], d1: ["DB"] };
+const broker = createBroker({
+  db: ":memory:",
+  token: "native-utf8",
+  bindings: { ...bindings, resources: {} },
+});
+const brokerServer = listen(broker, "127.0.0.1", 0);
+let server: ReturnType<typeof Bun.spawn> | undefined;
+try {
+  const porffor = await ensurePorffor({ cacheRoot: workdir });
+  const node = Bun.which("node");
+  if (!node) throw new Error("native UTF-8 test needs node on PATH");
+
+  const [core, transport] = await Promise.all([
+    readFile(preludePath, "utf8"),
+    readFile(transportPath("broker"), "utf8"),
+  ]);
+  assert(core.includes(TRANSPORT_MARKER), "runtime prelude is missing its transport marker");
+  const generated = join(workdir, "utf8.generated.js");
+  const binary = join(workdir, "utf8-native");
+  await writeFile(generated, wrapNativeFetchHandler(handler, core.replace(TRANSPORT_MARKER, transport), {}, bindings));
+
+  const compile = Bun.spawn([node, resolve(porffor, "runtime/index.js"), "native", generated, "-o", binary, "-s", "-O0"], {
+    cwd: workdir,
+    env: process.env,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    compile.exited,
+    new Response(compile.stdout).text(),
+    new Response(compile.stderr).text(),
+  ]);
+  assert.equal(exitCode, 0, `Porffor native compile failed:\n${stderr}\n${stdout}`);
+
+  const port = freePort();
+  server = Bun.spawn([binary], {
+    cwd: workdir,
+    env: {
+      ...process.env,
+      PORT: String(port),
+      SB_BROKER_PORT: String(brokerServer.port),
+      SB_BROKER_TOKEN: "native-utf8",
+    },
+    stdout: "ignore",
+    stderr: "inherit",
+  });
+  let response: Response | undefined;
+  for (let attempt = 0; attempt < 50; attempt++) {
+    try {
+      response = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(2000) });
+      break;
+    } catch {
+      if (server.exitCode !== null) break;
+      await Bun.sleep(100);
+    }
+  }
+  assert(response, "compiled UTF-8 handler did not start");
+  assert.equal(response.status, 200);
+  // SAFETY: the handler above is the only producer of this body and always
+  // returns exactly these five fields; deepEqual below checks every one.
+  const body = (await response.json()) as { kv: string; kvLength: number; d1: string; d1Length: number; error: string };
+  assert.deepEqual(
+    { kv: body.kv, kvLength: body.kvLength, d1: body.d1, d1Length: body.d1Length },
+    { kv: text, kvLength: text.length, d1: text, d1Length: text.length },
+  );
+  assert.match(body.error, /tabellà/);
+  console.log("Non-ASCII KV, D1 and error text round-trip through the broker");
+} finally {
+  if (server) {
+    server.kill(9);
+    await server.exited;
+  }
+  brokerServer.stop();
+  broker.close();
+  await rm(workdir, { recursive: true, force: true });
+}