@sproutboat/runtime 0.2.0 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @sproutboat/runtime
 
+## 0.3.0
+
+### Minor Changes
+
+- Standalone runtime: client IP, D1 online backup, and a prepared-statement cache.
+  
+  - `request.cf.clientIp` is populated from the connection's remote address
+    (`x-sb-remote-addr`, appended by the server and stripped from client input),
+    with `SB_TRUSTED_PROXIES` resolving `X-Forwarded-For` behind a reverse proxy.
+    IPv4-mapped IPv6 peers are folded to dotted form. (baronunread/sproutboat#163)
+  - `env.<D1>.backup(name?)` — a Sproutboat extension: an online, integrity-checked
+    single-file snapshot via `VACUUM INTO`, on both the embedded and broker
+    transports. New `d1.backup` op. (baronunread/sproutboat#164)
+  - The embedded transport caches prepared statements per database (FIFO, 32/db)
+    instead of recompiling the SQL on every binding op. (baronunread/sproutboat#155)
+
 ## 0.2.0
 
 ### Minor Changes

package/README.md

@@ -1,6 +1,33 @@
 # `@sproutboat/runtime`
 
-Single source of truth for the sprout runtime: the binding/trigger wrapper,
-handler source validation, both transports (broker and embedded), and the
+The sprout runtime as a unit: the binding and trigger wrapper, handler
+source validation, both transports (broker and embedded), and the
 native-fetch prelude. These move as one because `wrap.ts` locates the
 prelude and transports by file URL beside itself.
+
+```sh
+bun add @sproutboat/runtime
+```
+
+```ts
+import { preludePath, transportPath, wrapNativeFetchHandler } from "@sproutboat/runtime";
+import { validateHttpSyncSource } from "@sproutboat/runtime";
+```
+
+## API
+
+- `wrapNativeFetchHandler(...)` turns a user's `export default { fetch }`
+  into a native-fetch module with bindings, triggers, and env installed.
+  `Bindings`, `EMPTY_BINDINGS`, `readVarsFromEnv`, `readBindingsFromEnv`,
+  and `neutraliseExports` travel with it.
+- `preludePath` / `transportPath(transport)` locate the prelude and the
+  `broker` / `embedded` transports on disk. The prelude is read as text
+  and prepended before compilation, never imported.
+- `TRANSPORT_MARKER`, `BASELINE_COMPATIBILITY_DATE`, `Transport` type.
+- `validateHttpSyncSource(source)` checks a bundled handler module
+  (`SourceValidation`), used by project checks on both sides.
+
+The prelude shims the Web API surface handlers expect (URL, Response,
+crypto random values, binding shims, trigger dispatch) and is identical
+for both transports, which is what lets one conformance suite hold them
+honest.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/runtime",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "license": "MIT",
   "type": "module",
   "main": "./src/index.ts",

package/src/client-ip.test.js

@@ -0,0 +1,66 @@
+// #163 — the client-IP resolution helpers live inline in native-fetch-prelude.js
+// (the prelude is prepended as text and cannot import), so this test lifts that
+// one pure-JS block out and exercises it directly. If the block moves or its
+// boundary markers change, this fails loudly rather than silently testing nothing.
+import { expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+const src = readFileSync(fileURLToPath(new URL("./native-fetch-prelude.js", import.meta.url)), "utf8");
+const start = src.indexOf("function __sbParseHex(");
+const end = src.indexOf("globalThis.__sbEntry");
+if (start === -1 || end === -1 || end < start) throw new Error("client-ip helper block not found in prelude");
+
+let env = {};
+// oxlint-disable-next-line no-function-constructor -- lifting a pure block out of the text-only prelude for test
+const load = new Function(
+  "__envMap",
+  `${src.slice(start, end)}
+   const __sbEnv = (k) => __envMap[k];
+   return { __sbNormalizeIp, __sbIpInCidr, __sbIpTrusted, __sbClientIp, __sbSplitList };`,
+);
+const H = load(new Proxy({}, { get: (_, k) => env[k] }));
+
+const req = (headers) => ({ headers: { get: (k) => headers[k.toLowerCase()] ?? null } });
+
+test("__sbNormalizeIp folds IPv4-mapped IPv6 to dotted quad", () => {
+  expect(H.__sbNormalizeIp("0000:0000:0000:0000:0000:ffff:7f00:0001")).toBe("127.0.0.1");
+  expect(H.__sbNormalizeIp("::ffff:198.51.100.9")).toBe("198.51.100.9");
+  expect(H.__sbNormalizeIp("203.0.113.7")).toBe("203.0.113.7");
+  expect(H.__sbNormalizeIp("2001:db8::1")).toBe("2001:db8::1"); // real IPv6 untouched
+  expect(H.__sbNormalizeIp("")).toBe("");
+});
+
+test("__sbIpInCidr does IPv4 range math without sign errors above 2^31", () => {
+  expect(H.__sbIpInCidr("10.1.2.3", "10.0.0.0/8")).toBe(true);
+  expect(H.__sbIpInCidr("11.0.0.1", "10.0.0.0/8")).toBe(false);
+  expect(H.__sbIpInCidr("203.0.113.7", "203.0.113.7")).toBe(true); // bare IP
+  expect(H.__sbIpInCidr("192.168.1.1", "0.0.0.0/0")).toBe(true);
+  // 224.x is > 2^31 as a uint32 — the divide-not-mask path.
+  expect(H.__sbIpInCidr("224.0.0.5", "224.0.0.0/24")).toBe(true);
+  expect(H.__sbIpInCidr("224.0.1.5", "224.0.0.0/24")).toBe(false);
+});
+
+test("__sbClientIp: no trusted proxies means the peer wins, XFF ignored", () => {
+  env = {};
+  const r = req({ "x-sb-remote-addr": "::ffff:203.0.113.7", "x-forwarded-for": "1.2.3.4" });
+  expect(H.__sbClientIp(r)).toBe("203.0.113.7");
+});
+
+test("__sbClientIp: a trusted peer resolves the rightmost untrusted XFF hop", () => {
+  env = { SB_TRUSTED_PROXIES: "127.0.0.0/8, 10.0.0.0/8" };
+  expect(H.__sbClientIp(req({ "x-sb-remote-addr": "127.0.0.1", "x-forwarded-for": "203.0.113.7, 10.1.2.3" }))).toBe(
+    "203.0.113.7",
+  );
+  // every hop trusted → fall back to the peer
+  expect(H.__sbClientIp(req({ "x-sb-remote-addr": "127.0.0.1", "x-forwarded-for": "10.9.9.9" }))).toBe("127.0.0.1");
+  // mapped-v6 entry in the chain is normalised
+  expect(
+    H.__sbClientIp(req({ "x-sb-remote-addr": "10.0.0.1", "x-forwarded-for": "::ffff:198.51.100.9, 10.1.2.3" })),
+  ).toBe("198.51.100.9");
+});
+
+test("__sbClientIp: an untrusted peer cannot use XFF even if it sends one", () => {
+  env = { SB_TRUSTED_PROXIES: "10.0.0.0/8" };
+  expect(H.__sbClientIp(req({ "x-sb-remote-addr": "198.51.100.9", "x-forwarded-for": "1.1.1.1" }))).toBe("198.51.100.9");
+});

package/src/native-fetch-prelude.js

@@ -421,6 +421,14 @@ function __sbMakeD1(dbName) {
       __sbRpc("d1.exec", { db: dbName, sql: String(sql) });
       return { count: (String(sql).match(/;/g) || []).length, duration: 0 };
     },
+    // Sproutboat extension (not in CF Workers D1): an online, integrity-checked
+    // snapshot of this database. Standalone only for now — the broker transport
+    // returns an error until hosted backup (#139) covers it. `name` lands under
+    // `<data-dir>/backups/`; omit it for a timestamped default. See #164.
+    backup(name) {
+      const r = __sbRpc("d1.backup", { db: dbName, name: name == null ? "" : String(name) });
+      return { path: r.path, bytes: r.bytes };
+    },
   };
 }
 
@@ -844,9 +852,119 @@ function __sbTriggerAuthed(request) {
   return request.headers.get("x-sb-token") === want;
 }
 
+// #163 — the connection's remote address, for `request.cf.clientIp`.
+//
+// `x-sb-remote-addr` is the TCP peer, appended by the server (see
+// patches/UPSTREAM.md #163) and stripped from anything a client sends, so it
+// cannot be forged. With no proxy in front, that is the client. Behind one, set
+// `SB_TRUSTED_PROXIES` to a comma-separated list of trusted CIDRs (or bare IPs):
+// when the peer is trusted, the client is the rightmost `x-forwarded-for` entry
+// that is not itself a trusted hop.
+//
+// ponytail: IPv4 CIDR ranges + exact-string match (which covers a bare IPv6).
+// An IPv6 *prefix* in SB_TRUSTED_PROXIES matches nothing — widen __sbIpInCidr if
+// a deployment ever fronts its sprout with an IPv6 proxy range.
+function __sbParseHex(s) {
+  let n = 0;
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i);
+    let d = -1;
+    if (c >= 48 && c <= 57) d = c - 48;
+    else if (c >= 97 && c <= 102) d = c - 87;
+    else if (c >= 65 && c <= 70) d = c - 55;
+    if (d < 0) return -1;
+    n = n * 16 + d;
+  }
+  return n;
+}
+// uWS hands back IPv4-mapped IPv6 for IPv4 clients on a dual-stack socket
+// (`::ffff:1.2.3.4`, or fully expanded `0:0:0:0:0:ffff:0102:0304`). Fold those
+// to the dotted IPv4 so CIDR matching and `clientIp` see a plain address.
+function __sbNormalizeIp(ip) {
+  const s = String(ip || "");
+  if (s.indexOf(":") === -1) return s;
+  const low = s.toLowerCase();
+  const dotted = low.indexOf("::ffff:");
+  if (dotted === 0 && low.indexOf(".") !== -1) return low.slice(7);
+  const g = low.split(":");
+  if (g.length === 8) {
+    let mapped = __sbParseHex(g[5]) === 0xffff;
+    for (let i = 0; i < 5; i++) if (__sbParseHex(g[i] || "0") !== 0) mapped = false;
+    if (mapped) {
+      const hi = __sbParseHex(g[6] || "0");
+      const lo = __sbParseHex(g[7] || "0");
+      if (hi >= 0 && lo >= 0) {
+        return ((hi / 256) | 0) + "." + (hi & 0xff) + "." + ((lo / 256) | 0) + "." + (lo & 0xff);
+      }
+    }
+  }
+  return s;
+}
+function __sbTrim(s) {
+  let a = 0;
+  let b = s.length;
+  while (a < b && (s.charCodeAt(a) === 32 || s.charCodeAt(a) === 9)) a++;
+  while (b > a && (s.charCodeAt(b - 1) === 32 || s.charCodeAt(b - 1) === 9)) b--;
+  return s.slice(a, b);
+}
+function __sbSplitList(raw) {
+  const out = [];
+  const parts = String(raw || "").split(",");
+  for (let i = 0; i < parts.length; i++) {
+    const v = __sbTrim(parts[i]);
+    if (v) out.push(v);
+  }
+  return out;
+}
+function __sbIpToLong(ip) {
+  const p = String(ip).split(".");
+  if (p.length !== 4) return -1;
+  let n = 0;
+  for (let i = 0; i < 4; i++) {
+    if (p[i] === "" || p[i].length > 3) return -1;
+    const o = Number(p[i]);
+    if (!(o >= 0 && o <= 255) || o !== (o | 0)) return -1;
+    n = n * 256 + o;
+  }
+  return n;
+}
+function __sbIpInCidr(ip, cidr) {
+  const slash = cidr.indexOf("/");
+  if (slash === -1) return ip === cidr; // bare IP: exact match, covers IPv6
+  const addr = __sbIpToLong(ip);
+  const base = __sbIpToLong(cidr.slice(0, slash));
+  const bits = Number(cidr.slice(slash + 1));
+  if (addr < 0 || base < 0 || !(bits >= 0 && bits <= 32)) return false;
+  if (bits === 0) return true;
+  if (bits === 32) return addr === base;
+  // Divide rather than mask: a 32-bit `&` in JS is signed and would miscompare
+  // addresses above 2^31.
+  const size = Math.pow(2, 32 - bits);
+  return Math.floor(addr / size) === Math.floor(base / size);
+}
+function __sbIpTrusted(ip, list) {
+  for (let i = 0; i < list.length; i++) if (__sbIpInCidr(ip, list[i])) return true;
+  return false;
+}
+function __sbClientIp(request) {
+  const peer = __sbNormalizeIp(request.headers.get("x-sb-remote-addr") || "");
+  const trusted = __sbSplitList(__sbEnv("SB_TRUSTED_PROXIES"));
+  if (!trusted.length || !__sbIpTrusted(peer, trusted)) return peer;
+  const xff = __sbSplitList(request.headers.get("x-forwarded-for"));
+  for (let i = xff.length - 1; i >= 0; i--) {
+    const hop = __sbNormalizeIp(xff[i]);
+    if (!__sbIpTrusted(hop, trusted)) return hop;
+  }
+  return peer;
+}
+
 globalThis.__sbEntry = function (handlers, request) {
   const trigger = request.headers.get("x-sb-trigger");
   if (!trigger) {
+    // #163 — expose the resolved client IP the Workers way, before the handler runs.
+    const __cf = request.cf || {};
+    __cf.clientIp = __sbClientIp(request);
+    request.cf = __cf;
     // #28 — per-invocation CPU time. One fetch turn per process (serial), so the
     // process CPU delta across the handler is this invocation's CPU.
     // ponytail: serial-turn assumption; revisit if the profile ever allows

package/src/transport-embedded.js

@@ -35,6 +35,7 @@ extern int sqlite3_prepare_v2(sqlite3*, const char*, int, sqlite3_stmt**, const
 extern int sqlite3_step(sqlite3_stmt*);
 extern int sqlite3_finalize(sqlite3_stmt*);
 extern int sqlite3_reset(sqlite3_stmt*);
+extern int sqlite3_clear_bindings(sqlite3_stmt*);
 extern int sqlite3_column_count(sqlite3_stmt*);
 extern int sqlite3_column_type(sqlite3_stmt*, int);
 extern const unsigned char* sqlite3_column_text(sqlite3_stmt*, int);
@@ -50,6 +51,7 @@ extern const void* sqlite3_column_blob(sqlite3_stmt*, int);
 extern int sqlite3_changes(sqlite3*);
 extern int64_t sqlite3_last_insert_rowid(sqlite3*);
 extern const char* sqlite3_errmsg(sqlite3*);
+extern int sqlite3_close(sqlite3*);
 
 #define SB_SQLITE_ROW 100
 #define SB_SQLITE_DONE 101
@@ -103,6 +105,42 @@ static int sb_db_for(const char* path) {
   return sb_db_count++;
 }
 
+// #155 — a prepared-statement cache per database. Every binding op (KV, D1, DO
+// storage, queue, AE) funnels through sb_sql_run, which used to prepare and
+// finalize on every call — measured at ~0.6 ms/op, that compile step was most
+// of an op's cost. Keyed by exact SQL text; reset and re-bound on reuse.
+//
+// ponytail: FIFO eviction over 32 slots, linear scan, no lock (the runtime
+// serves one turn at a time). Move to LRU + a hash if a real app keeps >32 hot
+// statements per db, or if execution ever overlaps. Entries are never finalized
+// on shutdown — sb_db_for never closes a handle, the process owns it for life.
+#define SB_STMT_CACHE 32
+typedef struct { char* sql; sqlite3_stmt* st; } sb_cached_stmt;
+static sb_cached_stmt sb_stmt_cache[SB_MAX_DB][SB_STMT_CACHE];
+static int sb_stmt_fifo[SB_MAX_DB];
+
+static sqlite3_stmt* sb_stmt_get(int idx, sqlite3* db, const char* sql) {
+  sb_cached_stmt* slots = sb_stmt_cache[idx];
+  for (int i = 0; i < SB_STMT_CACHE; i++) {
+    if (slots[i].sql && strcmp(slots[i].sql, sql) == 0) {
+      sqlite3_reset(slots[i].st);
+      sqlite3_clear_bindings(slots[i].st);
+      return slots[i].st;
+    }
+  }
+  sqlite3_stmt* st = 0;
+  if (sqlite3_prepare_v2(db, sql, -1, &st, 0) != 0 || !st) return 0;
+  int slot = sb_stmt_fifo[idx];
+  sb_stmt_fifo[idx] = (slot + 1) % SB_STMT_CACHE;
+  if (slots[slot].st) sqlite3_finalize(slots[slot].st);
+  free(slots[slot].sql);
+  size_t n = strlen(sql);
+  slots[slot].sql = (char*)malloc(n + 1);
+  if (slots[slot].sql) memcpy(slots[slot].sql, sql, n + 1);
+  slots[slot].st = st;
+  return st;
+}
+
 // --- a growable output buffer, for building the reply JSON ------------------
 typedef struct { char* p; size_t len; size_t cap; } sb_buf;
 static void sb_buf_need(sb_buf* b, size_t extra) {
@@ -319,8 +357,8 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
   int idx = sb_db_for(path);
   if (idx < 0) { sb_puts(&b, "{\"ok\":false,\"error\":\"cannot open database\"}"); return b.p; }
   sqlite3* db = sb_dbs[idx];
-  sqlite3_stmt* st = 0;
-  if (sqlite3_prepare_v2(db, sql, -1, &st, 0) != 0 || !st) {
+  sqlite3_stmt* st = sb_stmt_get(idx, db, sql);
+  if (!st) {
     sb_puts(&b, "{\"ok\":false,\"error\":");
     const char* m = sqlite3_errmsg(db);
     sb_putjson(&b, m, strlen(m));
@@ -361,7 +399,9 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
   char tail[96];
   int k = snprintf(tail, 96, "],\"changes\":%d,\"rowid\":%lld}", sqlite3_changes(db), (long long)sqlite3_last_insert_rowid(db));
   sb_put(&b, tail, (size_t)k);
-  sqlite3_finalize(st);
+  // Reset, don't finalize: the statement stays compiled in sb_stmt_cache. Reset
+  // now (not just on next reuse) so a SELECT stops holding its read transaction.
+  sqlite3_reset(st);
   if (rc != SB_SQLITE_DONE && rc != SB_SQLITE_ROW) {
     free(b.p);
     sb_buf e = { 0, 0, 0 };
@@ -374,6 +414,67 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
   return b.p;
 }
 
+// #164 — an online, consistent single-file snapshot of a database, via
+// VACUUM INTO. Runs against the live db with no downtime (SQLite holds a read
+// transaction for the copy) and writes a fresh, defragmented file with no WAL
+// sidecars — exactly what you copy off the box. The copy is then reopened and
+// PRAGMA integrity_check'd, so a bad backup fails here, not on restore.
+// Returns {"ok":true,"bytes":n} or {"ok":false,"error":"..."}.
+static char* sb_sql_backup(const char* src_path, const char* dest_path) {
+  sb_buf b = { 0, 0, 0 };
+  int idx = sb_db_for(src_path);
+  if (idx < 0) { sb_puts(&b, "{\"ok\":false,\"error\":\"cannot open database\"}"); return b.p; }
+  sqlite3* db = sb_dbs[idx];
+
+  sb_mkdirs(dest_path);
+  unlink(dest_path); // VACUUM INTO refuses to overwrite an existing file
+
+  sqlite3_stmt* st = 0;
+  if (sqlite3_prepare_v2(db, "VACUUM INTO ?1", -1, &st, 0) != 0 || !st) {
+    sb_puts(&b, "{\"ok\":false,\"error\":");
+    const char* m = sqlite3_errmsg(db);
+    sb_putjson(&b, m, strlen(m));
+    sb_puts(&b, "}");
+    return b.p;
+  }
+  sqlite3_bind_text(st, 1, dest_path, -1, SB_SQLITE_TRANSIENT);
+  int rc = sqlite3_step(st);
+  sqlite3_finalize(st);
+  if (rc != SB_SQLITE_DONE) {
+    sb_puts(&b, "{\"ok\":false,\"error\":");
+    const char* m = sqlite3_errmsg(db);
+    sb_putjson(&b, m, strlen(m));
+    sb_puts(&b, "}");
+    return b.p;
+  }
+
+  sqlite3* chk = 0;
+  int ok = 0;
+  if (sqlite3_open(dest_path, &chk) == 0 && chk) {
+    sqlite3_stmt* cs = 0;
+    if (sqlite3_prepare_v2(chk, "PRAGMA integrity_check", -1, &cs, 0) == 0 && cs) {
+      if (sqlite3_step(cs) == SB_SQLITE_ROW) {
+        const unsigned char* r = sqlite3_column_text(cs, 0);
+        ok = r && strcmp((const char*)r, "ok") == 0;
+      }
+      sqlite3_finalize(cs);
+    }
+  }
+  if (chk) sqlite3_close(chk);
+  if (!ok) {
+    unlink(dest_path);
+    sb_puts(&b, "{\"ok\":false,\"error\":\"integrity check failed on the backup\"}");
+    return b.p;
+  }
+
+  struct stat sbuf;
+  long long bytes = stat(dest_path, &sbuf) == 0 ? (long long)sbuf.st_size : -1;
+  char head[64];
+  int k = snprintf(head, sizeof(head), "{\"ok\":true,\"bytes\":%lld}", bytes);
+  sb_put(&b, head, (size_t)k);
+  return b.p;
+}
+
 // --- outbound HTTP + HTTPS ---------------------------------------------------
 // Plain sockets for http, BearSSL for https, with the Mozilla root set compiled
 // in (see src/bearssl.ts). Both directions share the request builder and the
@@ -894,6 +995,33 @@ function __sbSqlScriptRaw(path, sql) {
   return res;
 }
 
+// #164 — VACUUM INTO a fresh file. Two strings in, JSON out. Same pattern as
+// __sbSqlScriptRaw.
+// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
+function __sbD1BackupRaw(src, dest) {
+  let res = "";
+  // oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
+  Porffor.c`
+    const char* __s; size_t __sl; char* __so = 0;
+    porf_native_fetch_read_value(src, &__s, &__sl, &__so);
+    char* __src = (char*)malloc(__sl + 1); memcpy(__src, __s, __sl); __src[__sl] = 0;
+    if (__so) free(__so);
+
+    const char* __d; size_t __dl; char* __do = 0;
+    porf_native_fetch_read_value(dest, &__d, &__dl, &__do);
+    char* __dest = (char*)malloc(__dl + 1); memcpy(__dest, __d, __dl); __dest[__dl] = 0;
+    if (__do) free(__do);
+
+    char* __out = sb_sql_backup(__src, __dest);
+    free(__src); free(__dest);
+    if (__out) {
+      res = porf_box((f64)porf_native_fetch_alloc_bytestring(__out, strlen(__out)), 195);
+      free(__out);
+    }
+  `;
+  return res;
+}
+
 // Outbound HTTP. String params so C can read each directly; the JS side has
 // already split the URL and enforced the allowlist. `tlsFlag` is "1" or "0" —
 // a string like the rest, so the marshalling stays uniform.
@@ -1199,6 +1327,20 @@ function __sbEmbeddedDispatch(msg) {
     return { ok: true, results: one.results, meta: one.meta, success: true };
   }
 
+  if (op === "d1.backup") {
+    const src = __sbD1Path(String(msg.db));
+    // A caller-supplied name must be a plain basename; anything else falls back
+    // to the timestamped default, so nothing can climb out of backups/.
+    const given = String(msg.name || "");
+    const name = /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(given)
+      ? given
+      : String(msg.db) + "-" + new Date().toISOString().replace(/[:.]/g, "-") + ".sqlite";
+    const dest = __sbDir() + "/backups/" + name;
+    const reply = JSON.parse(__sbD1BackupRaw(src, dest));
+    if (reply.ok === false) throw new Error("d1 backup: " + reply.error);
+    return { ok: true, path: dest, bytes: reply.bytes };
+  }
+
   if (op === "r2.put") {
     const body = String(msg.body == null ? "" : msg.body);
     const etag = __sbHex(16);