@sproutboat/runtime 0.25.0 → 0.27.0

package/src/transport-embedded.js

@@ -419,9 +419,26 @@ static int sb_ltx_begin(sb_ltx_writer *w, const char *path, sb_ltx_state *s, uin
   memset(w, 0, sizeof(*w));
   w->hash = UINT64_MAX;
   w->page_size = s->page_size;
+  // Global temporary directories are outside a sandboxed worker's writable tree.
+  // After unlink, close (including process death) reclaims the scratch file.
+  char index_path[SB_LTX_PATH];
+  if (snprintf(index_path, sizeof(index_path), "%s/index-XXXXXX", s->root) >= sizeof(index_path))
+    return -1;
+  int index = mkstemp(index_path);
+  if (index < 0)
+    return -1;
+  sb_ltx_phase("index-created");
+  if (unlink(index_path)) {
+    close(index);
+    return -1;
+  }
+  w->index = fdopen(index, "w+b");
+  if (!w->index) {
+    close(index);
+    return -1;
+  }
   w->out = fopen(path, "wbx");
-  w->index = tmpfile();
-  if (!w->out || !w->index)
+  if (!w->out)
     return -1;
   unsigned char h[100] = {0};
   memcpy(h, "LTX1", 4);
@@ -1048,6 +1065,41 @@ done:
   return 1;
 }
 
+// The capture lock is already held. Remove only reserved, regular scratch files;
+// do not open them (a hard link could share the database's POSIX-locked inode).
+static int sb_ltx_sweep_indexes(const char *root) {
+  DIR *dir = opendir(root);
+  if (!dir)
+    return -1;
+  int rc = 0;
+  for (;;) {
+    errno = 0;
+    struct dirent *entry = readdir(dir);
+    if (!entry) {
+      if (errno)
+        rc = -1;
+      break;
+    }
+    const char *name = entry->d_name;
+    if (strlen(name) != 12 || strncmp(name, "index-", 6) ||
+        strspn(name + 6, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != 6)
+      continue;
+    struct stat st;
+    if (fstatat(dirfd(dir), name, &st, AT_SYMLINK_NOFOLLOW)) {
+      if (errno == ENOENT)
+        continue;
+      rc = -1;
+      break;
+    }
+    if (S_ISREG(st.st_mode) && unlinkat(dirfd(dir), name, 0) && errno != ENOENT) {
+      rc = -1;
+      break;
+    }
+  }
+  if (closedir(dir))
+    rc = -1;
+  return rc;
+}
 static int sb_ltx_start(int idx, sqlite3 *db, const char *path, int lock) {
   sb_ltx_state *s = &sb_ltx_states[idx];
   memset(s, 0, sizeof(*s));
@@ -1059,6 +1111,8 @@ static int sb_ltx_start(int idx, sqlite3 *db, const char *path, int lock) {
     return -1;
   if (mkdir(s->root, 0700) && errno != EEXIST)
     return -1;
+  if (sb_ltx_sweep_indexes(s->root))
+    return -1;
   char parent[SB_LTX_PATH];
   snprintf(parent, sizeof(parent), "%s", path);
   char *slash = strrchr(parent, '/');
@@ -2137,12 +2191,12 @@ static int sb_db_page_size(const char* path) {
   if (st) sqlite3_finalize(st);
   return size;
 }
-// Freeze (refuse writes to) or thaw a database for the quota. Returns 1 when the database
-// is under LTX capture, 0 when not, -1 when it cannot be opened.
+// Freeze (refuse writes to) or thaw a database for the quota; on < 0 only asks. Returns 1
+// when the database is under LTX capture, 0 when not, -1 when it cannot be opened.
 static int sb_db_quota_freeze(const char* path, int on) {
   int idx = sb_db_for(path);
   if (idx < 0) return -1;
-  sb_quota_frozen[idx] = on && sb_ltx_states[idx].db ? 1 : 0;
+  if (on >= 0) sb_quota_frozen[idx] = on && sb_ltx_states[idx].db ? 1 : 0;
   return sb_ltx_states[idx].db ? 1 : 0;
 }
 
@@ -3197,7 +3251,7 @@ function __sbD1DbRaw(path, what) {
     if (__po) free(__po);
     const char* __w; size_t __wl; char* __wo = 0;
     porf_native_fetch_read_value(what, &__w, &__wl, &__wo);
-    int __r = __wl && __w[0] == 'p' ? sb_db_page_size(__path) : sb_db_quota_freeze(__path, __wl && __w[0] == 'f');
+    int __r = __wl && __w[0] == 'p' ? sb_db_page_size(__path) : __wl && __w[0] == 'c' ? sb_db_quota_freeze(__path, -1) : sb_db_quota_freeze(__path, __wl && __w[0] == 'f');
     if (__wo) free(__wo);
     free(__path);
     char __n[16]; int __k = snprintf(__n, sizeof(__n), "%d", __r);
@@ -3455,11 +3509,31 @@ var __sbQuotaOps = {
 // A database under LTX capture is frozen instead: every write is refused, because even a
 // DELETE adds a file to the replication spool, which counts toward the quota.
 var __sbD1Capped = {};
+var __sbD1Captured = {};
+var __sbD1Checkpointed = {};
 function __sbD1Quota(path) {
-  const full = __sbQuotaFull();
+  // Whether capture owns this database, from the private helper (no SQL), before anything
+  // below depends on it. A database that cannot open (or whose capture failed) is left to
+  // the call itself, which reports why.
+  const status = Number(__sbD1DbRaw(path, "captured"));
+  if (status < 0) return;
+  __sbD1Captured[path] = status === 1;
+  let full = __sbQuotaFull();
+  // Over the quota, a database's -wal file can hold space the app has already freed (a
+  // DELETE, a VACUUM): checkpoint it into the database and truncate it, at most once a
+  // second, then measure again. Not under capture: LTX owns that database's checkpoints.
+  if (full && !__sbD1Captured[path] && Date.now() - (__sbD1Checkpointed[path] || 0) >= 1000) {
+    __sbD1Checkpointed[path] = Date.now();
+    JSON.parse(__sbSqlRaw(path, "PRAGMA wal_checkpoint(TRUNCATE)", "[]"));
+    __sbQuotaAt = -1;
+    full = __sbQuotaFull();
+  }
   if (__sbD1Capped[path] === full) return;
   __sbD1Capped[path] = full;
-  const captured = Number(__sbD1DbRaw(path, full ? "freeze" : "thaw")) === 1;
+  const frozen = Number(__sbD1DbRaw(path, full ? "freeze" : "thaw"));
+  if (frozen < 0) return;
+  const captured = frozen === 1;
+  __sbD1Captured[path] = captured;
   if (full && captured) return;
   const size = Number(__sbD1DbRaw(path, "page_size")) || 4096;
   const pages = full
@@ -3499,6 +3573,10 @@ function __sbEnsureSchema() {
   let hasKvBinary = false;
   for (let i = 0; i < kvColumns.length; i++) if (kvColumns[i][1] === "binary") hasKvBinary = true;
   if (!hasKvBinary) __sbSql(s, "ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0", []);
+  // A key's metadata (put(key, value, { metadata })), as JSON text; same column as the broker's.
+  let hasKvMetadata = false;
+  for (let i = 0; i < kvColumns.length; i++) if (kvColumns[i][1] === "metadata") hasKvMetadata = true;
+  if (!hasKvMetadata) __sbSql(s, "ALTER TABLE kv ADD COLUMN metadata TEXT", []);
   // #56 — no `body` column: an object's bytes live in their own file under
   // "<data-dir>/r2-blobs/" (sb_r2_put_c/sb_r2_get_c), never in the sqlite row.
   __sbSql(
@@ -3512,6 +3590,10 @@ function __sbEnsureSchema() {
   let hasBlobId = false;
   for (let i = 0; i < r2Columns.length; i++) if (r2Columns[i][1] === "blob_id") hasBlobId = true;
   if (!hasBlobId) __sbSql(s, "ALTER TABLE r2 ADD COLUMN blob_id TEXT", []);
+  // The MD5 of an object's bytes, its etag from compatibility date 2026-10-26 (r2.md5).
+  let hasR2Md5 = false;
+  for (let i = 0; i < r2Columns.length; i++) if (r2Columns[i][1] === "md5") hasR2Md5 = true;
+  if (!hasR2Md5) __sbSql(s, "ALTER TABLE r2 ADD COLUMN md5 TEXT", []);
   __sbSql(
     s,
     "CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
@@ -3583,6 +3665,51 @@ function __sbEnsureSchema() {
   );
 }
 
+// One page of r2.list for the shim from 2026-10-26: at most LIMIT entries after CURSOR,
+// keys under PREFIX, those holding DELIMITER after it grouped into delimitedPrefixes (a
+// group counts as one entry). Same rules as the broker's r2ListPage.
+// ponytail: with a delimiter every key after the cursor is read; an index on the
+// delimited prefix if buckets grow past ~100k keys.
+function __sbR2ListPage(store, msg) {
+  const prefix = String(msg.prefix || "");
+  const delimiter = String(msg.delimiter || "");
+  const limit = Math.min(Math.max(Math.floor(Number(msg.limit)) || 1000, 1), 1000);
+  const r = __sbSql(
+    store,
+    "SELECT key, size, etag, uploaded, http_json, custom_json, md5 FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 " +
+      "AND key > ?3 ORDER BY key" + (delimiter ? "" : " LIMIT " + (limit + 1)),
+    [msg.bucket, prefix, String(msg.cursor || "")],
+  );
+  const objects = [];
+  const delimitedPrefixes = [];
+  let last = "";
+  let truncated = false;
+  for (let i = 0; i < r.rows.length; i++) {
+    const row = r.rows[i];
+    const at = delimiter ? row[0].indexOf(delimiter, prefix.length) : -1;
+    const group = at < 0 ? "" : row[0].slice(0, at + delimiter.length);
+    if (!group || delimitedPrefixes[delimitedPrefixes.length - 1] !== group) {
+      if (objects.length + delimitedPrefixes.length >= limit) {
+        truncated = true;
+        break;
+      }
+      if (group) delimitedPrefixes.push(group);
+      else
+        objects.push({
+          key: row[0],
+          size: Number(row[1]),
+          etag: row[2],
+          uploaded: row[3],
+          httpMetadata: JSON.parse(row[4] || "{}"),
+          customMetadata: JSON.parse(row[5] || "{}"),
+          md5: row[6],
+        });
+    }
+    last = row[0];
+  }
+  return { ok: true, objects, delimitedPrefixes, truncated, cursor: truncated ? last : "" };
+}
+
 function __sbHex(n) {
   let out = "";
   const bytes = __sbRandomBytes(String(n));
@@ -3602,7 +3729,7 @@ function __sbEmbeddedDispatch(msg) {
   if (op === "ping") return { ok: true, op: "pong", echo: msg.msg };
 
   if (op === "kv.get") {
-    const r = __sbSql(store, "SELECT value, expires_at, binary FROM kv WHERE ns = ? AND key = ?", [
+    const r = __sbSql(store, "SELECT value, expires_at, binary, metadata FROM kv WHERE ns = ? AND key = ?", [
       msg.ns,
       msg.key,
     ]);
@@ -3612,7 +3739,14 @@ function __sbEmbeddedDispatch(msg) {
       __sbSql(store, "DELETE FROM kv WHERE ns = ? AND key = ?", [msg.ns, msg.key]);
       return { ok: true, found: false, value: null };
     }
-    return { ok: true, found: true, value: r.rows[0][0], binary: r.rows[0][2] === 1 };
+    return {
+      ok: true,
+      found: true,
+      value: r.rows[0][0],
+      binary: r.rows[0][2] === 1,
+      metadata: r.rows[0][3],
+      expiration: expiresAt == null ? null : Math.floor(expiresAt / 1000),
+    };
   }
   if (op === "kv.put") {
     // #58 — CF rejects expirationTtl under 60 seconds; matched here.
@@ -3626,8 +3760,8 @@ function __sbEmbeddedDispatch(msg) {
     }
     __sbSql(
       store,
-      "INSERT INTO kv (ns, key, value, expires_at, binary) VALUES (?1,?2,?3,?4,?5) ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5",
-      [msg.ns, msg.key, msg.value, expiresAt, msg.binary === true ? 1 : 0],
+      "INSERT INTO kv (ns, key, value, expires_at, binary, metadata) VALUES (?1,?2,?3,?4,?5,?6) ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5, metadata = ?6",
+      [msg.ns, msg.key, msg.value, expiresAt, msg.binary === true ? 1 : 0, msg.metadata == null ? null : String(msg.metadata)],
     );
     return { ok: true };
   }
@@ -3636,14 +3770,28 @@ function __sbEmbeddedDispatch(msg) {
     return { ok: true };
   }
   if (op === "kv.list") {
+    // The prefix matches exactly: LIKE ignored case and read "_" and "%" in it as wildcards.
+    // With a limit (the shim from 2026-10-25), a page of entries after the cursor (the last
+    // key of the previous page), with each key's expiration and metadata; without one,
+    // every key, as before.
+    const prefix = msg.prefix || "";
+    const paged = msg.limit != null;
+    const limit = paged ? Math.max(0, Math.floor(Number(msg.limit))) : -1;
     const r = __sbSql(
       store,
-      "SELECT key FROM kv WHERE ns = ? AND key LIKE ? || '%' AND (expires_at IS NULL OR expires_at > ?) ORDER BY key",
-      [msg.ns, msg.prefix || "", Date.now()],
+      "SELECT key, expires_at, metadata FROM kv WHERE ns = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 " +
+        "AND (expires_at IS NULL OR expires_at > ?4) ORDER BY key LIMIT ?5",
+      [msg.ns, prefix, paged && msg.cursor ? String(msg.cursor) : "", Date.now(), paged ? limit + 1 : -1],
     );
     const keys = [];
-    for (let i = 0; i < r.rows.length; i++) keys.push(r.rows[i][0]);
-    return { ok: true, keys };
+    const entries = [];
+    const count = paged ? Math.min(r.rows.length, limit) : r.rows.length;
+    for (let i = 0; i < count; i++) {
+      keys.push(r.rows[i][0]);
+      entries.push({ name: r.rows[i][0], expiration: r.rows[i][1] == null ? null : Math.floor(r.rows[i][1] / 1000), metadata: r.rows[i][2] });
+    }
+    const complete = !paged || r.rows.length <= limit;
+    return { ok: true, keys, entries, list_complete: complete, cursor: complete ? "" : keys[keys.length - 1] };
   }
 
   if (op === "assets.get") {
@@ -3973,7 +4121,7 @@ function __sbEmbeddedDispatch(msg) {
     const wantsBody = op === "r2.get";
     const r = __sbSql(
       store,
-      "SELECT size, etag, uploaded, http_json, custom_json, blob_id FROM r2 WHERE bucket = ? AND key = ?",
+      "SELECT size, etag, uploaded, http_json, custom_json, blob_id, md5 FROM r2 WHERE bucket = ? AND key = ?",
       [msg.bucket, msg.key],
     );
     if (!r.rows.length) return { ok: true, found: false };
@@ -3989,6 +4137,7 @@ function __sbEmbeddedDispatch(msg) {
         uploaded: row[2],
         httpMetadata: JSON.parse(row[3] || "{}"),
         customMetadata: JSON.parse(row[4] || "{}"),
+        md5: row[6],
       },
       blobId: row[5] || "",
       body: wantsBody
@@ -4011,10 +4160,23 @@ function __sbEmbeddedDispatch(msg) {
       );
     return { ok: true };
   }
+  if (op === "r2.md5") {
+    // Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
+    __sbSql(store, "UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4", [
+      String(msg.md5),
+      msg.bucket,
+      msg.key,
+      String(msg.etag),
+    ]);
+    return { ok: true };
+  }
   if (op === "r2.list") {
+    // The prefix matches exactly at every date: LIKE ignored case and read "_" and "%" as
+    // wildcards. With `page` (the shim from 2026-10-26), a page after the cursor.
+    if (msg.page) return __sbR2ListPage(store, msg);
     const r = __sbSql(
       store,
-      "SELECT key, size, etag, uploaded FROM r2 WHERE bucket = ? AND key LIKE ? || '%' ORDER BY key",
+      "SELECT key, size, etag, uploaded FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 ORDER BY key",
       [msg.bucket, msg.prefix || ""],
     );
     const objects = [];

package/test/ltx-restore.ts

@@ -0,0 +1,110 @@
+// LTX chain restore, the same independent checker as native-ltx.ts (copied, so that test
+// stays as it is): every file's CRCs, checksums and TXIDs verified, pages applied in order.
+import assert from "node:assert/strict";
+import { readFile, readdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+// Independent CRC baseline in Bun. This intentionally uses BigInt, unlike C.
+const table: bigint[] = [];
+for (let i = 0; i < 256; i++) {
+  let c = BigInt(i);
+  for (let j = 0; j < 8; j++) c = (c >> 1n) ^ (c & 1n ? 0xd800000000000000n : 0n);
+  table.push(c);
+}
+function crc(data: Uint8Array, init = 0xffffffffffffffffn) {
+  let c = init;
+  for (const b of data) c = table[Number((c ^ BigInt(b)) & 255n)] ^ (c >> 8n);
+  return c;
+}
+const flag = 0x8000000000000000n,
+  mask = 0xffffffffffffffffn;
+function pageCrc(pgno: number, page: Uint8Array) {
+  const p = new Uint8Array(4);
+  new DataView(p.buffer).setUint32(0, pgno);
+  return flag | (crc(page, crc(p)) ^ mask);
+}
+
+export async function restore(epoch: string, output: string) {
+  const files = (await readdir(epoch)).filter((n) => n.endsWith(".ltx")).sort();
+  let db = new Uint8Array(0),
+    checksum = 0n,
+    txid = 0n;
+  for (const file of files) {
+    const b = new Uint8Array(await readFile(join(epoch, file)));
+    const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
+    assert.equal(new TextDecoder().decode(b.subarray(0, 4)), "LTX1");
+    const pageSize = v.getUint32(8),
+      commit = v.getUint32(12);
+    assert.equal(v.getBigUint64(16), txid + 1n);
+    assert.equal(v.getBigUint64(40), checksum);
+    const next = new Uint8Array(commit * pageSize);
+    next.set(db.subarray(0, next.length));
+    db = next;
+    let offset = 100,
+      hash = crc(b.subarray(0, 100));
+    const entries = [];
+    for (;;) {
+      const begin = offset,
+        pgno = v.getUint32(offset),
+        flags = v.getUint16(offset + 4);
+      hash = crc(b.subarray(offset, offset + 6), hash);
+      offset += 6;
+      if (!pgno) {
+        assert.equal(flags, 0);
+        break;
+      }
+      assert.equal(flags, 1);
+      assert(pgno <= commit);
+      const size = v.getUint32(offset);
+      hash = crc(b.subarray(offset, offset + 4), hash);
+      offset += 4;
+      const stop = offset + size;
+      assert(stop <= b.length);
+      assert.equal(b[offset++], 0xf0);
+      let literals = 15,
+        byte;
+      do {
+        byte = b[offset++];
+        literals += byte;
+      } while (byte === 255);
+      assert.equal(literals, pageSize);
+      assert.equal(offset + pageSize, stop);
+      const page = b.subarray(offset, stop);
+      db.set(page, (pgno - 1) * pageSize);
+      hash = crc(page, hash);
+      offset = stop;
+      entries.push({ pgno, offset: begin, size: stop - begin });
+    }
+    const indexStart = offset;
+    function varint() {
+      let n = 0n,
+        s = 0n;
+      for (let i = 0; i < 10; i++) {
+        assert(offset < b.length);
+        const c = b[offset++];
+        n |= BigInt(c & 127) << s;
+        if (c < 128) return Number(n);
+        s += 7n;
+      }
+      throw new Error("varint overflow");
+    }
+    for (const entry of entries) {
+      assert.equal(varint(), entry.pgno);
+      assert.equal(varint(), entry.offset);
+      assert.equal(varint(), entry.size);
+    }
+    assert.equal(varint(), 0);
+    assert.equal(v.getBigUint64(offset), BigInt(offset - indexStart));
+    offset += 8;
+    hash = crc(b.subarray(indexStart, offset + 8), hash);
+    checksum = flag;
+    for (let p = 1; p <= commit; p++)
+      checksum = flag | (checksum ^ pageCrc(p, db.subarray((p - 1) * pageSize, p * pageSize)));
+    assert.equal(v.getBigUint64(offset), checksum, file);
+    assert.equal(v.getBigUint64(offset + 8), flag | (hash ^ mask), file);
+    assert.equal(offset + 16, b.length);
+    txid = v.getBigUint64(24);
+  }
+  await writeFile(output, db);
+  return { files: files.length, txid };
+}