@sproutboat/runtime 0.24.0 → 0.26.0

package/test/native-d1.ts

@@ -1,4 +1,6 @@
 import assert from "node:assert/strict";
+import { Database } from "bun:sqlite";
+import { restore } from "./ltx-restore";
 import { closeSync, openSync, readFileSync, readdirSync, readSync, writeSync } from "node:fs";
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -47,6 +49,8 @@ int main(void) {
     if (!strncmp(line, ":break ", 7)) { line[strcspn(line, "\\t")] = 0; char p[1024]; snprintf(p, sizeof(p), "%s.ltx/position.tmp", line + 7); mkdir(p, 0700); puts("{}"); fflush(stdout); continue; }
     char* sql = strchr(line, '\\t'); if (!sql) return 2; *sql++ = 0;
     char* params = strchr(sql, '\\t'); if (!params) return 2; *params++ = 0;
+    if (!strncmp(line, ":db ", 4)) { int r = sql[0] == 'p' ? sb_db_page_size(line + 4) : sb_db_quota_freeze(line + 4, sql[0] == 'c' ? -1 : sql[0] == 'f'); printf("{\\"n\\":%d}\\n", r); fflush(stdout); continue; }
+    if (!strncmp(line, ":bytes ", 7)) { line[strcspn(line, "\\t")] = 0; printf("{\\"n\\":%lld}\\n", sb_dir_bytes(line + 7, 0)); fflush(stdout); continue; }
     // ":script PATH" runs SQL as a script (sb_sql_script, what exec() uses).
     char* out = !strncmp(line, ":script ", 8) ? sb_sql_script(line + 8, sql) : sb_sql_run(line, sql, params); puts(out); fflush(stdout); free(out);
   }
@@ -147,7 +151,9 @@ int main(void) {
   const toDriver = openSync(fifoIn, "w"),
     fromDriver = openSync(fifoOut, "r");
   let pending = "";
+  let driverClosed = false;
   const sqlRaw = (path: string, sql: string, params: string) => {
+    if (driverClosed) throw new Error("original D1 driver is closed");
     writeSync(toDriver, `${path}\t${sql}\t${params}\n`);
     const chunk = Buffer.alloc(65536);
     while (!pending.includes("\n")) {
@@ -172,6 +178,89 @@ int main(void) {
     "__sbSqlScriptRaw",
     `${transport.slice(opsStart, opsEnd)}\nreturn { __sbD1Run, __sbD1Batch, __sbD1Open, __sbD1ExecLines };`,
   )(sqlRaw, sql, scriptRaw);
+  // The storage quota, lifted from "The storage quota" to __sbStore, for DIR and LIMIT.
+  const quotaStart = transport.indexOf("// The storage quota: an app's data directory");
+  const quotaEnd = transport.indexOf("function __sbStore() {");
+  assert(quotaStart > 0 && quotaEnd > quotaStart, "quota block moved");
+  const liftQuota = (dir: string, limit: number) =>
+    new Function(
+      "__sbEnv",
+      "__sbDir",
+      "__sbDirBytesRaw",
+      "__sbD1DbRaw",
+      "__sbSql",
+      "__sbSqlRaw",
+      `${transport.slice(quotaStart, quotaEnd)}\nreturn { __sbQuotaCheck, __sbD1Quota, __sbQuotaFull, expire() { __sbQuotaAt = -1; } };`,
+    )(
+      (name: string) => (name === "SB_STORAGE_MAX_BYTES" ? String(limit) : ""),
+      () => dir,
+      (d: string) => String(JSON.parse(sqlRaw(":bytes " + d, "", "")).n),
+      (path: string, what: string) => String(JSON.parse(sqlRaw(":db " + path, what, "")).n),
+      sql,
+      sqlRaw,
+    );
+  const dirBytes = (d: string) => Number(JSON.parse(sqlRaw(":bytes " + d, "", "")).n);
+  // Another driver process (a restarted worker), with its own FIFOs and lifted blocks.
+  const connect = (tag: string) => {
+    const inPath = join(work, "in-" + tag),
+      outPath = join(work, "out-" + tag);
+    assert.equal(Bun.spawnSync(["mkfifo", inPath, outPath]).exitCode, 0);
+    const child = Bun.spawn(["sh", "-c", `exec "${binary}" < "${inPath}" > "${outPath}"`], {
+      env: { ...process.env, SB_LTX_ENABLE: "1", SB_LTX_DATABASE: "LTX" },
+    });
+    const to = openSync(inPath, "w"),
+      from = openSync(outPath, "r");
+    let buffered = "";
+    const raw = (path: string, text: string, params: string) => {
+      writeSync(to, `${path}\t${text}\t${params}\n`);
+      const chunk = Buffer.alloc(65536);
+      while (!buffered.includes("\n")) {
+        const n = readSync(from, chunk, 0, chunk.length, null);
+        if (n <= 0) throw new Error("driver exited");
+        buffered += chunk.toString("utf8", 0, n);
+      }
+      const line = buffered.slice(0, buffered.indexOf("\n"));
+      buffered = buffered.slice(line.length + 1);
+      return line;
+    };
+    const run = (path: string, text: string, params: unknown[]) => {
+      const r = JSON.parse(raw(path, text, JSON.stringify(params)));
+      if (r.ok === false) throw new Error("sqlite: " + r.error);
+      return r;
+    };
+    // oxlint-disable-next-line no-function-constructor -- lifting a block out of the text-only transport
+    const lifted = new Function(
+      "__sbSqlRaw",
+      "__sbSql",
+      "__sbSqlScriptRaw",
+      `${transport.slice(opsStart, opsEnd)}\nreturn { __sbD1Run, __sbD1Batch, __sbD1Open, __sbD1ExecLines };`,
+    )(raw, run, (path: string, text: string) => raw(":script " + path, text, "[]"));
+    const lift = (dir: string, limit: number) =>
+      // oxlint-disable-next-line no-function-constructor -- lifting a block out of the text-only transport
+      new Function(
+        "__sbEnv",
+        "__sbDir",
+        "__sbDirBytesRaw",
+        "__sbD1DbRaw",
+        "__sbSql",
+        "__sbSqlRaw",
+        `${transport.slice(quotaStart, quotaEnd)}\nreturn { __sbQuotaCheck, __sbD1Quota, __sbQuotaFull };`,
+      )(
+        (name: string) => (name === "SB_STORAGE_MAX_BYTES" ? String(limit) : ""),
+        () => dir,
+        (d: string) => String(JSON.parse(raw(":bytes " + d, "", "")).n),
+        (path: string, what: string) => String(JSON.parse(raw(":db " + path, what, "")).n),
+        run,
+        raw,
+      );
+    return {
+      sqlRaw: raw,
+      ops: lifted,
+      liftQuota: lift,
+      exited: child.exited,
+      close: () => (closeSync(to), closeSync(from)),
+    };
+  };
   // exec() from 2026-10-24: line by line, all or nothing, the failing line named.
   const ex = join(work, "data/d1/EXEC.sqlite");
   ops.__sbD1ExecLines(ex, ["CREATE TABLE x (id INTEGER)", "INSERT INTO x VALUES (1); INSERT INTO x VALUES (2);"]);
@@ -257,6 +346,67 @@ int main(void) {
     true,
   );
   assert.equal(ltxFiles(), beforeBatch + 1, "a batch is one LTX commit");
+  // With capture on, an authorization failure says so (it read "not an error" once capture
+  // had run), and the quota works: its page size comes from capture, not from the
+  // page_size pragma the capture authorizer refuses.
+  assert.throws(
+    () => ops.__sbD1Run(ltx, "PRAGMA page_size", [], true),
+    (e: Error) => e.message === "D1_ERROR: not authorized: SQLITE_AUTH",
+  );
+  assert.throws(
+    () => ops.__sbD1ExecLines(ltx, ["PRAGMA journal_mode = DELETE"]),
+    (e: Error) => e.message === "D1_EXEC_ERROR: Error in line 1: PRAGMA journal_mode = DELETE: not authorized",
+  );
+  const data = join(work, "data");
+  const roomy = liftQuota(data, dirBytes(data) + (1 << 20));
+  roomy.__sbD1Quota(ltx);
+  ops.__sbD1Run(ltx, "INSERT INTO e VALUES (10, 'under the quota')", [], true);
+  assert.equal(ops.__sbD1Run(ltx, "SELECT v FROM e WHERE id = 10", [], true).rows[0][0], "under the quota");
+  // Over the quota with capture on, every write is refused, a same-size UPDATE and a DELETE
+  // too (each would add an LTX file to the spool), and the spool stops growing; reads and
+  // read-only batches still run. Under the quota again, writes resume.
+  const tight = liftQuota(data, 1);
+  tight.__sbD1Quota(ltx);
+  const spool = ltxFiles();
+  const QUOTA =
+    "D1_ERROR: storage quota exceeded: writes are refused while this app's data, including its replication log, is over its quota: SQLITE_FULL";
+  for (const write of [
+    "INSERT INTO e VALUES (11, 'x')",
+    "UPDATE e SET v = 'y' WHERE id = 10",
+    "DELETE FROM e WHERE id = 10",
+  ])
+    assert.throws(
+      () => ops.__sbD1Run(ltx, write, [], true),
+      (e: Error) => e.message === QUOTA,
+      write,
+    );
+  assert.throws(
+    () => ops.__sbD1ExecLines(ltx, ["UPDATE e SET v = 'z'"]),
+    (e: Error) => e.message.startsWith("D1_EXEC_ERROR: Error in line 1: UPDATE e SET v = 'z': storage quota exceeded"),
+  );
+  assert.throws(
+    () =>
+      ops.__sbD1Batch(
+        ltx,
+        [
+          { sql: "SELECT 1", params: [] },
+          { sql: "DELETE FROM e", params: [] },
+        ],
+        true,
+      ),
+    (e: Error) => e.message === QUOTA,
+  );
+  assert.equal(
+    ops.__sbD1Batch(ltx, [{ sql: "SELECT count(*) FROM e", params: [] }], true).length,
+    1,
+    "a read-only batch runs",
+  );
+  assert.equal(ops.__sbD1Run(ltx, "SELECT v FROM e WHERE id = 10", [], true).rows[0][0], "under the quota");
+  assert.equal(ltxFiles(), spool, "no LTX file while over the quota");
+  const again = liftQuota(data, dirBytes(data) + (1 << 20));
+  again.__sbD1Quota(ltx);
+  ops.__sbD1Run(ltx, "UPDATE e SET v = 'thawed' WHERE id = 10", [], true);
+  assert.equal(ltxFiles(), spool + 1, "writes, and capture, resume under the quota");
   // Break capture just before the batch's RELEASE.
   // oxlint-disable-next-line no-function-constructor -- lifting a block out of the text-only transport
   const breaking = new Function(
@@ -264,10 +414,14 @@ int main(void) {
     "__sbSql",
     "__sbSqlScriptRaw",
     `${transport.slice(opsStart, opsEnd)}\nreturn { __sbD1Run, __sbD1Batch };`,
-  )((path: string, text: string, params: string) => {
-    if (text === "RELEASE sb_d1_batch") sqlRaw(":break " + path, "", "");
-    return sqlRaw(path, text, params);
-  }, sql, scriptRaw);
+  )(
+    (path: string, text: string, params: string) => {
+      if (text === "RELEASE sb_d1_batch") sqlRaw(":break " + path, "", "");
+      return sqlRaw(path, text, params);
+    },
+    sql,
+    scriptRaw,
+  );
   const LTX_FAILED =
     "D1_ERROR: LTX capture failed: the transaction may be committed locally but is not durable in the replication log; writes are refused until restart: SQLITE_IOERR";
   assert.throws(
@@ -286,12 +440,101 @@ int main(void) {
   );
   assert.throws(
     () => ops.__sbD1ExecLines(ltx, ["INSERT INTO e VALUES (5, 'e')"]),
-    (e: Error) => e.message === "D1_ERROR: LTX capture failed: the transaction may be committed locally but is not durable in the replication log; writes are refused until restart: SQLITE_IOERR",
+    (e: Error) =>
+      e.message ===
+      "D1_ERROR: LTX capture failed: the transaction may be committed locally but is not durable in the replication log; writes are refused until restart: SQLITE_IOERR",
     "exec() after a capture failure",
   );
+  const qdir = join(work, "quota");
+  const qdb = join(qdir, "d1/Q.sqlite");
+  const quota = liftQuota(qdir, 262144);
+  ops.__sbD1Run(qdb, "CREATE TABLE q (b BLOB)", [], true);
+  quota.__sbD1Quota(qdb);
+  quota.__sbQuotaCheck("KV put()");
+  // Fill to the limit: a database is capped at the quota, so growth past it fails with
+  // SQLITE_FULL; then the directory is over, a KV write is refused, and reads and deletes
+  // keep working. VACUUM gives the space back and writes resume.
+  let rows = 0;
+  assert.throws(
+    () => {
+      for (; rows < 64; rows++) ops.__sbD1Run(qdb, "INSERT INTO q VALUES (zeroblob(65536))", [], true);
+    },
+    (e: Error) => e.message === "D1_ERROR: database or disk is full: SQLITE_FULL",
+  );
+  assert(rows >= 2 && rows < 5, "filled to about 256 KiB: " + rows);
+  quota.expire();
+  assert.equal(quota.__sbQuotaFull(), true);
+  assert.throws(
+    () => quota.__sbQuotaCheck("KV put()"),
+    /^Error: KV put\(\) failed: storage quota exceeded \(this app stores \d+ bytes; the limit is 262144\)$/,
+  );
+  quota.__sbD1Quota(qdb);
+  assert.equal(ops.__sbD1Run(qdb, "SELECT count(*) FROM q", [], true).rows[0][0], rows, "reads still work");
+  ops.__sbD1Run(qdb, "DELETE FROM q", [], true);
+  ops.__sbD1Run(qdb, "VACUUM", [], true);
+  // No checkpoint by the app: over the quota the runtime checkpoints and truncates the WAL
+  // (once a second) before measuring again, so DELETE and VACUUM alone make room.
+  await Bun.sleep(1000);
+  quota.expire();
+  quota.__sbD1Quota(qdb);
+  quota.__sbQuotaCheck("KV put()");
+  ops.__sbD1Run(qdb, "INSERT INTO q VALUES (zeroblob(65536))", [], true);
+  driverClosed = true;
   closeSync(toDriver);
   closeSync(fromDriver);
   await driver.exited;
+
+  // A restart that finds the app over its quota, capture on: the first D1 call of the new
+  // worker (process) already freezes. Reads work, writes are refused, the spool and its
+  // position stay exactly as they were, and the chain restores the database byte for byte.
+  // A capture failure still reads as one, over the quota too.
+  await rm(ltx + ".ltx/position.tmp", { recursive: true, force: true });
+  const second = connect("restart");
+  const quota2 = second.liftQuota(data, 1);
+  quota2.__sbD1Quota(ltx);
+  const spoolBefore = readdirSync(generation()).sort();
+  const positionBefore = readFileSync(ltx + ".ltx/position", "utf8");
+  const count = second.ops.__sbD1Run(ltx, "SELECT count(*) FROM e", [], true).rows[0][0];
+  for (const write of ["INSERT INTO e VALUES (20, 'x')", "UPDATE e SET v = v", "DELETE FROM e"])
+    assert.throws(
+      () => second.ops.__sbD1Run(ltx, write, [], true),
+      (e: Error) => e.message === QUOTA,
+      "restart, over the quota: " + write,
+    );
+  assert.deepEqual(readdirSync(generation()).sort(), spoolBefore, "no LTX file after a clean resume over the quota");
+  assert.equal(readFileSync(ltx + ".ltx/position", "utf8"), positionBefore, "the position is unchanged");
+  second.close();
+  await second.exited;
+  {
+    const live = new Database(ltx);
+    live.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    assert.equal(live.query("SELECT count(*) AS n FROM e").get()?.n, count);
+    live.close();
+    const restored = join(work, "restart-restored.sqlite");
+    await restore(generation(), restored);
+    assert.deepEqual(
+      await readFile(restored),
+      await readFile(ltx),
+      "the retained chain restores the database byte for byte",
+    );
+  }
+  const third = connect("failure");
+  const thirdBytes = Number(JSON.parse(third.sqlRaw(":bytes " + data, "", "")).n);
+  third.liftQuota(data, thirdBytes + (1 << 20)).__sbD1Quota(ltx);
+  const breaking3 = (path: string, text: string, params: string) => {
+    if (text.startsWith("INSERT INTO e VALUES (30")) third.sqlRaw(":break " + path, "", "");
+    return third.sqlRaw(path, text, params);
+  };
+  const brokenWrite = JSON.parse(breaking3(ltx, "INSERT INTO e VALUES (30, 'x')", "[]"));
+  assert.equal(brokenWrite.code, 10, "a capture failure reports SQLITE_IOERR");
+  const failed3 = JSON.parse(third.sqlRaw(ltx, "INSERT INTO e VALUES (31, 'y')", "[]"));
+  assert.equal(failed3.code, 10, "after a capture failure, writes report it");
+  third.liftQuota(data, 1).__sbD1Quota(ltx);
+  const failedOver = JSON.parse(third.sqlRaw(ltx, "INSERT INTO e VALUES (32, 'z')", "[]"));
+  assert.equal(failedOver.code, 10, "over the quota too, the capture failure is what a write reports");
+  assert.match(failedOver.error, /^LTX capture failed/);
+  third.close();
+  await third.exited;
   console.log(
     "native d1: sb_sql_run and the D1 ops pass (blob binds, atomic batch, strict counts, foreign keys, error text)",
   );

package/test/native-ltx.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert/strict";
-import { mkdtemp, readFile, readdir, rm, writeFile, stat } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { Database } from "bun:sqlite";
@@ -24,12 +24,18 @@ const source = `
 #include <sys/stat.h>
 #include <time.h>
 #include <signal.h>
+#include <errno.h>
+// Reproduce denial of global temporary-file access on every test platform.
+// The sproutd sandbox qualification separately exercises real Linux Landlock.
+static FILE *forbidden_tmpfile(void) { errno=EACCES; return NULL; }
+#define tmpfile forbidden_tmpfile
 #define SB_MAX_DB 16
 ${transport.slice(start, end)}
 int main(int argc,char **argv) {
   if(argc!=2)return 2;sb_ltx_compat=1;if(sb_ltx_register())return 3;
   char lockpath[1024];snprintf(lockpath,sizeof(lockpath),"%s.ltx.lock",argv[1]);
   int lock=open(lockpath,O_RDWR|O_CREAT,0600);if(lock<0||flock(lock,LOCK_EX|LOCK_NB))return 4;
+  if(getenv("SB_LTX_TEST_HOLD_LOCK")){puts("locked");fflush(stdout);getchar();close(lock);return 0;}
   sqlite3 *db=0;if(sqlite3_open_v2(argv[1],&db,6,"sb-ltx-bounded"))return 5;
   if(getenv("SB_LTX_TEST_AUTO_VACUUM"))sqlite3_exec(db,"PRAGMA auto_vacuum=INCREMENTAL;",0,0,0);
   if(sqlite3_exec(db,"PRAGMA journal_mode=WAL;PRAGMA cache_size=-2000;",0,0,0)||sb_ltx_start(0,db,argv[1],lock))return 6;
@@ -153,6 +159,13 @@ async function restore(epoch: string, output: string) {
   await writeFile(output, db);
   return { files: files.length, txid };
 }
+async function noIndexScratch(path: string) {
+  const entries = await readdir(path + ".ltx", { withFileTypes: true });
+  assert(
+    !entries.some((entry) => entry.isFile() && /^index-[a-zA-Z0-9]{6}$/.test(entry.name)),
+    "index scratch file leaked",
+  );
+}
 async function run(path: string, sql: string, extra: Record<string, string> = {}) {
   const child = Bun.spawn([binary, path], {
     env: { ...process.env, ...extra },
@@ -169,6 +182,7 @@ async function run(path: string, sql: string, extra: Record<string, string> = {}
   ]);
   assert.equal(code, 0, error);
   assert(out.startsWith("ready\n"), out);
+  await noIndexScratch(path);
   return out.split("\n").slice(1).filter(Boolean);
 }
 async function stoppedCommit(path: string, phase: string, count = 1) {
@@ -212,6 +226,10 @@ async function stoppedCommit(path: string, phase: string, count = 1) {
   }
   child.kill("SIGKILL");
   await child.exited;
+  if (phase === "index-created") {
+    const entries = await readdir(path + ".ltx", { withFileTypes: true });
+    assert.equal(entries.filter((entry) => entry.isFile() && /^index-[a-zA-Z0-9]{6}$/.test(entry.name)).length, 1);
+  } else await noIndexScratch(path);
   assert(stopped, "did not reach crash boundary " + phase);
   return epoch;
 }
@@ -241,6 +259,45 @@ try {
   const cleanPosition = await readFile(root + "/position", "utf8");
   assert(cleanPosition.split("\n")[2].startsWith("C "));
   const finishedBefore = await readdir(join(root, epoch));
+  // SIGKILL can land between mkstemp and unlink. Only the next lock owner may
+  // discard that reserved scratch name; unrelated files and links stay intact.
+  const strayIndex = join(root, "index-abc123");
+  await writeFile(strayIndex, "stale index");
+  const unrelatedIndex = join(root, "index-not-scratch");
+  await writeFile(unrelatedIndex, "keep");
+  const indexDirectory = join(root, "index-DIR123");
+  await mkdir(indexDirectory);
+  await writeFile(join(indexDirectory, "keep"), "keep directory");
+  const outside = join(work, "outside-index-target");
+  await writeFile(outside, "keep target");
+  const indexLink = join(root, "index-LNK123");
+  await symlink(outside, indexLink);
+  const owner = Bun.spawn([binary, path], {
+    stdin: "pipe",
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...process.env, SB_LTX_TEST_HOLD_LOCK: "1" },
+  });
+  const lockReader = owner.stdout.getReader();
+  try {
+    const ready = await lockReader.read();
+    assert.equal(new TextDecoder().decode(ready.value), "locked\n");
+    const blocked = Bun.spawnSync([binary, path], { stdin: "ignore" });
+    assert.equal(blocked.exitCode, 4, "non-owner cannot initialize capture");
+    assert.equal(await readFile(strayIndex, "utf8"), "stale index", "non-owner must not sweep indexes");
+  } finally {
+    owner.stdin.end();
+    await owner.exited;
+    lockReader.releaseLock();
+  }
+  await run(path, "SELECT 1;\n");
+  assert(!(await readdir(root)).includes("index-abc123"), "resume must reclaim stale index");
+  assert.equal(await readFile(unrelatedIndex, "utf8"), "keep");
+  assert.equal(await readFile(join(indexDirectory, "keep"), "utf8"), "keep directory");
+  assert.equal(await readFile(indexLink, "utf8"), "keep target");
+  assert.equal(await readFile(outside, "utf8"), "keep target");
+  assert.equal(await readFile(root + "/position", "utf8"), cleanPosition);
+  assert.deepEqual(await readdir(join(root, epoch)), finishedBefore);
   for (let n = 0; n < 5; n++) {
     await run(path, "SELECT 1;\n");
     assert.equal(await readFile(root + "/position", "utf8"), cleanPosition, "idle restart must not rewrite position");
@@ -361,6 +418,7 @@ try {
   failedDb.close();
   const phases = [
     "wal-committed",
+    "index-created",
     "ltx-written",
     "ltx-fsynced",
     "ltx-published",
@@ -408,7 +466,7 @@ try {
   assert.equal(pdb.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
   pdb.close();
   console.log(
-    "PASS restart: 10 SIGKILL durability boundaries, matching-prefix resume, orphan/missing/reset fallback, pending committed WAL tail captured before serving",
+    "PASS restart: 11 SIGKILL durability boundaries, matching-prefix resume, orphan/missing/reset fallback, pending committed WAL tail captured before serving",
   );
   console.log(
     "PASS native LTX: 102-file chain, independent CRC/index checks, byte-exact restore, integrity, hard WAL ceiling and oversized transaction rollback",