@sproutboat/runtime 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/LTX_REPLICATION.md +19 -4
- package/package.json +1 -1
- package/src/kv-shim.test.js +124 -0
- package/src/native-fetch-prelude.js +121 -1
- package/src/transport-embedded.js +279 -14
- package/test/ltx-restore.ts +110 -0
- package/test/native-d1.ts +248 -5
- package/test/native-ltx.ts +60 -2
|
@@ -419,9 +419,26 @@ static int sb_ltx_begin(sb_ltx_writer *w, const char *path, sb_ltx_state *s, uin
|
|
|
419
419
|
memset(w, 0, sizeof(*w));
|
|
420
420
|
w->hash = UINT64_MAX;
|
|
421
421
|
w->page_size = s->page_size;
|
|
422
|
+
// Global temporary directories are outside a sandboxed worker's writable tree.
|
|
423
|
+
// After unlink, close (including process death) reclaims the scratch file.
|
|
424
|
+
char index_path[SB_LTX_PATH];
|
|
425
|
+
if (snprintf(index_path, sizeof(index_path), "%s/index-XXXXXX", s->root) >= sizeof(index_path))
|
|
426
|
+
return -1;
|
|
427
|
+
int index = mkstemp(index_path);
|
|
428
|
+
if (index < 0)
|
|
429
|
+
return -1;
|
|
430
|
+
sb_ltx_phase("index-created");
|
|
431
|
+
if (unlink(index_path)) {
|
|
432
|
+
close(index);
|
|
433
|
+
return -1;
|
|
434
|
+
}
|
|
435
|
+
w->index = fdopen(index, "w+b");
|
|
436
|
+
if (!w->index) {
|
|
437
|
+
close(index);
|
|
438
|
+
return -1;
|
|
439
|
+
}
|
|
422
440
|
w->out = fopen(path, "wbx");
|
|
423
|
-
w->
|
|
424
|
-
if (!w->out || !w->index)
|
|
441
|
+
if (!w->out)
|
|
425
442
|
return -1;
|
|
426
443
|
unsigned char h[100] = {0};
|
|
427
444
|
memcpy(h, "LTX1", 4);
|
|
@@ -1048,6 +1065,41 @@ done:
|
|
|
1048
1065
|
return 1;
|
|
1049
1066
|
}
|
|
1050
1067
|
|
|
1068
|
+
// The capture lock is already held. Remove only reserved, regular scratch files;
|
|
1069
|
+
// do not open them (a hard link could share the database's POSIX-locked inode).
|
|
1070
|
+
static int sb_ltx_sweep_indexes(const char *root) {
|
|
1071
|
+
DIR *dir = opendir(root);
|
|
1072
|
+
if (!dir)
|
|
1073
|
+
return -1;
|
|
1074
|
+
int rc = 0;
|
|
1075
|
+
for (;;) {
|
|
1076
|
+
errno = 0;
|
|
1077
|
+
struct dirent *entry = readdir(dir);
|
|
1078
|
+
if (!entry) {
|
|
1079
|
+
if (errno)
|
|
1080
|
+
rc = -1;
|
|
1081
|
+
break;
|
|
1082
|
+
}
|
|
1083
|
+
const char *name = entry->d_name;
|
|
1084
|
+
if (strlen(name) != 12 || strncmp(name, "index-", 6) ||
|
|
1085
|
+
strspn(name + 6, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != 6)
|
|
1086
|
+
continue;
|
|
1087
|
+
struct stat st;
|
|
1088
|
+
if (fstatat(dirfd(dir), name, &st, AT_SYMLINK_NOFOLLOW)) {
|
|
1089
|
+
if (errno == ENOENT)
|
|
1090
|
+
continue;
|
|
1091
|
+
rc = -1;
|
|
1092
|
+
break;
|
|
1093
|
+
}
|
|
1094
|
+
if (S_ISREG(st.st_mode) && unlinkat(dirfd(dir), name, 0) && errno != ENOENT) {
|
|
1095
|
+
rc = -1;
|
|
1096
|
+
break;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
if (closedir(dir))
|
|
1100
|
+
rc = -1;
|
|
1101
|
+
return rc;
|
|
1102
|
+
}
|
|
1051
1103
|
static int sb_ltx_start(int idx, sqlite3 *db, const char *path, int lock) {
|
|
1052
1104
|
sb_ltx_state *s = &sb_ltx_states[idx];
|
|
1053
1105
|
memset(s, 0, sizeof(*s));
|
|
@@ -1059,6 +1111,8 @@ static int sb_ltx_start(int idx, sqlite3 *db, const char *path, int lock) {
|
|
|
1059
1111
|
return -1;
|
|
1060
1112
|
if (mkdir(s->root, 0700) && errno != EEXIST)
|
|
1061
1113
|
return -1;
|
|
1114
|
+
if (sb_ltx_sweep_indexes(s->root))
|
|
1115
|
+
return -1;
|
|
1062
1116
|
char parent[SB_LTX_PATH];
|
|
1063
1117
|
snprintf(parent, sizeof(parent), "%s", path);
|
|
1064
1118
|
char *slash = strrchr(parent, '/');
|
|
@@ -1949,6 +2003,11 @@ void sb_r2_transfer_download_close(sb_r2_transfer_ctx* ctx) { if (ctx) { if (ctx
|
|
|
1949
2003
|
// the replication log, and the database refuses writes until the process restarts. Said as
|
|
1950
2004
|
// such (SQLITE_IOERR), never as SQLite's own message, which still reads "not an error".
|
|
1951
2005
|
#define SB_LTX_FAILED "LTX capture failed: the transaction may be committed locally but is not durable in the replication log; writes are refused until restart"
|
|
2006
|
+
// The storage quota, for a database under LTX capture: while the app is over its quota,
|
|
2007
|
+
// every write is refused (a same-size UPDATE or a DELETE still adds an LTX file to the
|
|
2008
|
+
// spool, which counts toward the quota). Reads and transaction control still run.
|
|
2009
|
+
#define SB_QUOTA_FULL "storage quota exceeded: writes are refused while this app's data, including its replication log, is over its quota"
|
|
2010
|
+
static unsigned char sb_quota_frozen[SB_MAX_DB];
|
|
1952
2011
|
static int sb_db_capture_failed(const char* path) {
|
|
1953
2012
|
for (int i = 0; i < sb_db_count; i++) if (strcmp(sb_db_names[i], path) == 0) return sb_ltx_states[i].failed;
|
|
1954
2013
|
return 0;
|
|
@@ -1963,13 +2022,18 @@ static char* sb_sql_script(const char* path, const char* sql) {
|
|
|
1963
2022
|
if (idx < 0 && sb_db_capture_failed(path)) { sb_puts(&b, "{\"ok\":false,\"error\":\"" SB_LTX_FAILED "\",\"code\":10}"); return b.p; }
|
|
1964
2023
|
if (idx < 0) { sb_puts(&b, "{\"ok\":false,\"error\":\"cannot open database\"}"); return b.p; }
|
|
1965
2024
|
char* err = 0;
|
|
1966
|
-
int script_rc = 0, capture_failed = 0;
|
|
2025
|
+
int script_rc = 0, capture_failed = 0, quota_full = 0, failure_code = 0;
|
|
2026
|
+
char failure[512] = { 0 };
|
|
1967
2027
|
if (sb_ltx_states[idx].db) {
|
|
1968
2028
|
const char *next = sql;
|
|
1969
2029
|
while (*next && !script_rc) {
|
|
1970
2030
|
sqlite3_stmt *st = 0; const char *tail = 0;
|
|
1971
2031
|
script_rc = sqlite3_prepare_v2(sb_dbs[idx], next, -1, &st, &tail);
|
|
2032
|
+
if (!script_rc && st && sb_quota_frozen[idx] && !sqlite3_stmt_readonly(st)) { sqlite3_finalize(st); quota_full = 1; script_rc = 13; break; }
|
|
1972
2033
|
if (!script_rc && st) { do { script_rc = sqlite3_step(st); } while (script_rc == SB_SQLITE_ROW); if (script_rc == SB_SQLITE_DONE) script_rc = 0; }
|
|
2034
|
+
// Keep the statement's error before capture runs statements of its own (it would
|
|
2035
|
+
// read "not an error" after them).
|
|
2036
|
+
if (script_rc && !failure[0]) { snprintf(failure, sizeof(failure), "%s", sqlite3_errmsg(sb_dbs[idx])); failure_code = sqlite3_extended_errcode(sb_dbs[idx]); }
|
|
1973
2037
|
if (st) sqlite3_finalize(st);
|
|
1974
2038
|
if (sb_ltx_after(idx)) { script_rc = 10; capture_failed = 1; }
|
|
1975
2039
|
if (!tail || tail <= next) break;
|
|
@@ -1977,10 +2041,12 @@ static char* sb_sql_script(const char* path, const char* sql) {
|
|
|
1977
2041
|
}
|
|
1978
2042
|
} else script_rc = sqlite3_exec(sb_dbs[idx], sql, 0, 0, &err);
|
|
1979
2043
|
if (script_rc != 0) {
|
|
1980
|
-
const char* m = capture_failed ? SB_LTX_FAILED : err ? err : sqlite3_errmsg(sb_dbs[idx]);
|
|
2044
|
+
const char* m = quota_full ? SB_QUOTA_FULL : capture_failed ? SB_LTX_FAILED : failure[0] ? failure : err ? err : sqlite3_errmsg(sb_dbs[idx]);
|
|
1981
2045
|
sb_puts(&b, "{\"ok\":false,\"error\":");
|
|
1982
2046
|
sb_putjson_text(&b, m, strlen(m));
|
|
1983
|
-
|
|
2047
|
+
char code[32];
|
|
2048
|
+
snprintf(code, sizeof(code), ",\"code\":%d}", quota_full ? 13 : capture_failed ? 10 : failure[0] ? failure_code : 0);
|
|
2049
|
+
sb_puts(&b, code);
|
|
1984
2050
|
return b.p;
|
|
1985
2051
|
}
|
|
1986
2052
|
sb_puts(&b, "{\"ok\":true}");
|
|
@@ -2021,6 +2087,10 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
|
|
|
2021
2087
|
sqlite3_stmt* st = sb_stmt_get(idx, db, sql);
|
|
2022
2088
|
if (!st) return sb_sql_error(db, 0, -1, strict ? (int)sizeof(SB_D1_STRICT) - 1 : 0);
|
|
2023
2089
|
int bound = params && *params ? sb_bind_params(st, params) : 0;
|
|
2090
|
+
if (sb_quota_frozen[idx] && !sqlite3_stmt_readonly(st)) {
|
|
2091
|
+
sqlite3_reset(st);
|
|
2092
|
+
return sb_sql_error(db, SB_QUOTA_FULL, 13, 0);
|
|
2093
|
+
}
|
|
2024
2094
|
if (strict && bound != sqlite3_bind_parameter_count(st)) {
|
|
2025
2095
|
sqlite3_reset(st);
|
|
2026
2096
|
return sb_sql_error(db, "Wrong number of parameter bindings for SQL query.", 0, 0);
|
|
@@ -2071,16 +2141,65 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
|
|
|
2071
2141
|
// Reset, don't finalize: the statement stays compiled in sb_stmt_cache. Reset
|
|
2072
2142
|
// now (not just on next reuse) so a SELECT stops holding its read transaction.
|
|
2073
2143
|
sqlite3_reset(st);
|
|
2144
|
+
// The statement's own error, kept before LTX capture runs: capture executes statements
|
|
2145
|
+
// of its own, after which SQLite reports "not an error".
|
|
2146
|
+
char failure[512] = { 0 };
|
|
2147
|
+
int failure_code = 0;
|
|
2148
|
+
if (rc != SB_SQLITE_DONE && rc != SB_SQLITE_ROW) {
|
|
2149
|
+
snprintf(failure, sizeof(failure), "%s", sqlite3_errmsg(db));
|
|
2150
|
+
failure_code = sqlite3_extended_errcode(db);
|
|
2151
|
+
}
|
|
2074
2152
|
const int capture_failed = sb_ltx_after(idx) != 0;
|
|
2075
2153
|
if (capture_failed) rc = 10;
|
|
2076
2154
|
if (rc != SB_SQLITE_DONE && rc != SB_SQLITE_ROW) {
|
|
2077
2155
|
free(b.p);
|
|
2078
2156
|
if (capture_failed) return sb_sql_error(db, SB_LTX_FAILED, 10, 0);
|
|
2079
|
-
return sb_sql_error(db,
|
|
2157
|
+
return sb_sql_error(db, failure, failure_code, 0);
|
|
2080
2158
|
}
|
|
2081
2159
|
return b.p;
|
|
2082
2160
|
}
|
|
2083
2161
|
|
|
2162
|
+
// The storage quota (docs: an app's data directory is capped): bytes in regular files under
|
|
2163
|
+
// dir, symlinks not followed. Bounded by depth; an unreadable entry counts as 0.
|
|
2164
|
+
static long long sb_dir_bytes(const char* dir, int depth) {
|
|
2165
|
+
DIR* d = opendir(dir);
|
|
2166
|
+
if (!d || depth > 16) { if (d) closedir(d); return 0; }
|
|
2167
|
+
long long total = 0;
|
|
2168
|
+
struct dirent* e;
|
|
2169
|
+
char path[4096];
|
|
2170
|
+
while ((e = readdir(d))) {
|
|
2171
|
+
if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) continue;
|
|
2172
|
+
if (snprintf(path, sizeof(path), "%s/%s", dir, e->d_name) >= (int)sizeof(path)) continue;
|
|
2173
|
+
struct stat st;
|
|
2174
|
+
if (lstat(path, &st)) continue;
|
|
2175
|
+
if (S_ISREG(st.st_mode)) total += (long long)st.st_size;
|
|
2176
|
+
else if (S_ISDIR(st.st_mode)) total += sb_dir_bytes(path, depth + 1);
|
|
2177
|
+
}
|
|
2178
|
+
closedir(d);
|
|
2179
|
+
return total;
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
// A database's page size without SQL: under LTX capture the authorizer refuses every
|
|
2183
|
+
// page_size pragma (the getter too), and capture records the size itself.
|
|
2184
|
+
static int sb_db_page_size(const char* path) {
|
|
2185
|
+
int idx = sb_db_for(path);
|
|
2186
|
+
if (idx < 0) return 0;
|
|
2187
|
+
if (sb_ltx_states[idx].db) return (int)sb_ltx_states[idx].page_size;
|
|
2188
|
+
sqlite3_stmt* st = 0;
|
|
2189
|
+
int size = 0;
|
|
2190
|
+
if (sqlite3_prepare_v2(sb_dbs[idx], "PRAGMA page_size", -1, &st, 0) == 0 && st && sqlite3_step(st) == SB_SQLITE_ROW) size = sqlite3_column_int(st, 0);
|
|
2191
|
+
if (st) sqlite3_finalize(st);
|
|
2192
|
+
return size;
|
|
2193
|
+
}
|
|
2194
|
+
// Freeze (refuse writes to) or thaw a database for the quota; on < 0 only asks. Returns 1
|
|
2195
|
+
// when the database is under LTX capture, 0 when not, -1 when it cannot be opened.
|
|
2196
|
+
static int sb_db_quota_freeze(const char* path, int on) {
|
|
2197
|
+
int idx = sb_db_for(path);
|
|
2198
|
+
if (idx < 0) return -1;
|
|
2199
|
+
if (on >= 0) sb_quota_frozen[idx] = on && sb_ltx_states[idx].db ? 1 : 0;
|
|
2200
|
+
return sb_ltx_states[idx].db ? 1 : 0;
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2084
2203
|
// #164 — an online, consistent single-file snapshot of a database, via
|
|
2085
2204
|
// VACUUM INTO. Runs against the live db with no downtime (SQLite holds a read
|
|
2086
2205
|
// transaction for the copy) and writes a fresh, defragmented file with no WAL
|
|
@@ -3103,6 +3222,44 @@ function __sbSqlScriptRaw(path, sql) {
|
|
|
3103
3222
|
return res;
|
|
3104
3223
|
}
|
|
3105
3224
|
|
|
3225
|
+
// The bytes under a directory, as a decimal string (sb_dir_bytes).
|
|
3226
|
+
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
3227
|
+
function __sbDirBytesRaw(dir) {
|
|
3228
|
+
let res = "";
|
|
3229
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
3230
|
+
Porffor.c`
|
|
3231
|
+
const char* __d; size_t __dl; char* __do = 0;
|
|
3232
|
+
porf_native_fetch_read_value(dir, &__d, &__dl, &__do);
|
|
3233
|
+
char* __dir = (char*)malloc(__dl + 1); memcpy(__dir, __d, __dl); __dir[__dl] = 0;
|
|
3234
|
+
if (__do) free(__do);
|
|
3235
|
+
char __n[32]; int __k = snprintf(__n, sizeof(__n), "%lld", sb_dir_bytes(__dir, 0));
|
|
3236
|
+
free(__dir);
|
|
3237
|
+
res = porf_box((f64)porf_native_fetch_alloc_bytestring(__n, (size_t)__k), 195);
|
|
3238
|
+
`;
|
|
3239
|
+
return res;
|
|
3240
|
+
}
|
|
3241
|
+
|
|
3242
|
+
// sb_db_page_size and sb_db_quota_freeze, for the quota (a path and a "0"/"1" in, a number out).
|
|
3243
|
+
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
3244
|
+
function __sbD1DbRaw(path, what) {
|
|
3245
|
+
let res = "";
|
|
3246
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
3247
|
+
Porffor.c`
|
|
3248
|
+
const char* __p; size_t __pl; char* __po = 0;
|
|
3249
|
+
porf_native_fetch_read_value(path, &__p, &__pl, &__po);
|
|
3250
|
+
char* __path = (char*)malloc(__pl + 1); memcpy(__path, __p, __pl); __path[__pl] = 0;
|
|
3251
|
+
if (__po) free(__po);
|
|
3252
|
+
const char* __w; size_t __wl; char* __wo = 0;
|
|
3253
|
+
porf_native_fetch_read_value(what, &__w, &__wl, &__wo);
|
|
3254
|
+
int __r = __wl && __w[0] == 'p' ? sb_db_page_size(__path) : __wl && __w[0] == 'c' ? sb_db_quota_freeze(__path, -1) : sb_db_quota_freeze(__path, __wl && __w[0] == 'f');
|
|
3255
|
+
if (__wo) free(__wo);
|
|
3256
|
+
free(__path);
|
|
3257
|
+
char __n[16]; int __k = snprintf(__n, sizeof(__n), "%d", __r);
|
|
3258
|
+
res = porf_box((f64)porf_native_fetch_alloc_bytestring(__n, (size_t)__k), 195);
|
|
3259
|
+
`;
|
|
3260
|
+
return res;
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3106
3263
|
// #164 — VACUUM INTO a fresh file. Two strings in, JSON out. Same pattern as
|
|
3107
3264
|
// __sbSqlScriptRaw.
|
|
3108
3265
|
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
@@ -3306,6 +3463,84 @@ function __sbDir() {
|
|
|
3306
3463
|
}
|
|
3307
3464
|
return __sbDataDir;
|
|
3308
3465
|
}
|
|
3466
|
+
// The storage quota: an app's data directory holds at most SB_STORAGE_MAX_BYTES (sproutd
|
|
3467
|
+
// sets it per app; 10 GiB when unset, after D1's 10 GB per database). Past it, writes are
|
|
3468
|
+
// refused with an error that says so; reads and deletes still work, so an app can make
|
|
3469
|
+
// room. The size is measured at most once a second, and on a write's way in, so a write
|
|
3470
|
+
// can overshoot by what one second of writes adds. A D1 database is also capped at 10 GB.
|
|
3471
|
+
var __sbQuotaMax = -1;
|
|
3472
|
+
var __sbQuotaUsed = 0;
|
|
3473
|
+
var __sbQuotaAt = -1;
|
|
3474
|
+
function __sbQuotaLimit() {
|
|
3475
|
+
if (__sbQuotaMax < 0) {
|
|
3476
|
+
const v = Number(__sbEnv("SB_STORAGE_MAX_BYTES"));
|
|
3477
|
+
__sbQuotaMax = v > 0 ? v : 10737418240;
|
|
3478
|
+
}
|
|
3479
|
+
return __sbQuotaMax;
|
|
3480
|
+
}
|
|
3481
|
+
function __sbQuotaFull() {
|
|
3482
|
+
const now = Date.now();
|
|
3483
|
+
if (__sbQuotaAt < 0 || now - __sbQuotaAt >= 1000) {
|
|
3484
|
+
__sbQuotaUsed = Number(__sbDirBytesRaw(__sbDir())) || 0;
|
|
3485
|
+
__sbQuotaAt = now;
|
|
3486
|
+
}
|
|
3487
|
+
return __sbQuotaUsed >= __sbQuotaLimit();
|
|
3488
|
+
}
|
|
3489
|
+
function __sbQuotaCheck(what) {
|
|
3490
|
+
if (__sbQuotaFull())
|
|
3491
|
+
throw new Error(
|
|
3492
|
+
what + " failed: storage quota exceeded (this app stores " + __sbQuotaUsed + " bytes; the limit is " + __sbQuotaLimit() + ")",
|
|
3493
|
+
);
|
|
3494
|
+
}
|
|
3495
|
+
var __sbQuotaOps = {
|
|
3496
|
+
"kv.put": "KV put()",
|
|
3497
|
+
"do.storage.put": "Durable Object storage put()",
|
|
3498
|
+
"do.alarm.set": "Durable Object setAlarm()",
|
|
3499
|
+
"queue.send": "Queue send()",
|
|
3500
|
+
"queue.send_batch": "Queue sendBatch()",
|
|
3501
|
+
"cache.put": "Cache put()",
|
|
3502
|
+
"ae.write": "Analytics Engine writeDataPoint()",
|
|
3503
|
+
"r2.put": "R2 put()",
|
|
3504
|
+
"r2.multipart.create": "R2 createMultipartUpload()",
|
|
3505
|
+
"r2.multipart.complete": "R2 completeMultipartUpload()",
|
|
3506
|
+
};
|
|
3507
|
+
// D1 over the quota: pin each database at its current size (SQLITE_FULL on growth, so
|
|
3508
|
+
// reads and deletes still run); under it, cap it at 10 GB or the quota, whichever is less.
|
|
3509
|
+
// A database under LTX capture is frozen instead: every write is refused, because even a
|
|
3510
|
+
// DELETE adds a file to the replication spool, which counts toward the quota.
|
|
3511
|
+
var __sbD1Capped = {};
|
|
3512
|
+
var __sbD1Captured = {};
|
|
3513
|
+
var __sbD1Checkpointed = {};
|
|
3514
|
+
function __sbD1Quota(path) {
|
|
3515
|
+
// Whether capture owns this database, from the private helper (no SQL), before anything
|
|
3516
|
+
// below depends on it. A database that cannot open (or whose capture failed) is left to
|
|
3517
|
+
// the call itself, which reports why.
|
|
3518
|
+
const status = Number(__sbD1DbRaw(path, "captured"));
|
|
3519
|
+
if (status < 0) return;
|
|
3520
|
+
__sbD1Captured[path] = status === 1;
|
|
3521
|
+
let full = __sbQuotaFull();
|
|
3522
|
+
// Over the quota, a database's -wal file can hold space the app has already freed (a
|
|
3523
|
+
// DELETE, a VACUUM): checkpoint it into the database and truncate it, at most once a
|
|
3524
|
+
// second, then measure again. Not under capture: LTX owns that database's checkpoints.
|
|
3525
|
+
if (full && !__sbD1Captured[path] && Date.now() - (__sbD1Checkpointed[path] || 0) >= 1000) {
|
|
3526
|
+
__sbD1Checkpointed[path] = Date.now();
|
|
3527
|
+
JSON.parse(__sbSqlRaw(path, "PRAGMA wal_checkpoint(TRUNCATE)", "[]"));
|
|
3528
|
+
__sbQuotaAt = -1;
|
|
3529
|
+
full = __sbQuotaFull();
|
|
3530
|
+
}
|
|
3531
|
+
if (__sbD1Capped[path] === full) return;
|
|
3532
|
+
__sbD1Capped[path] = full;
|
|
3533
|
+
const frozen = Number(__sbD1DbRaw(path, full ? "freeze" : "thaw"));
|
|
3534
|
+
if (frozen < 0) return;
|
|
3535
|
+
const captured = frozen === 1;
|
|
3536
|
+
__sbD1Captured[path] = captured;
|
|
3537
|
+
if (full && captured) return;
|
|
3538
|
+
const size = Number(__sbD1DbRaw(path, "page_size")) || 4096;
|
|
3539
|
+
const pages = full
|
|
3540
|
+
? Number(__sbSql(path, "PRAGMA page_count", []).rows[0][0])
|
|
3541
|
+
: Math.floor(Math.min(__sbQuotaLimit(), 10000000000) / size);
|
|
3542
|
+
__sbSql(path, "PRAGMA max_page_count = " + Math.max(1, pages), []);
|
|
3543
|
+
}
|
|
3309
3544
|
function __sbStore() {
|
|
3310
3545
|
return __sbDir() + "/store.sqlite";
|
|
3311
3546
|
}
|
|
@@ -3338,6 +3573,10 @@ function __sbEnsureSchema() {
|
|
|
3338
3573
|
let hasKvBinary = false;
|
|
3339
3574
|
for (let i = 0; i < kvColumns.length; i++) if (kvColumns[i][1] === "binary") hasKvBinary = true;
|
|
3340
3575
|
if (!hasKvBinary) __sbSql(s, "ALTER TABLE kv ADD COLUMN binary INTEGER NOT NULL DEFAULT 0", []);
|
|
3576
|
+
// A key's metadata (put(key, value, { metadata })), as JSON text; same column as the broker's.
|
|
3577
|
+
let hasKvMetadata = false;
|
|
3578
|
+
for (let i = 0; i < kvColumns.length; i++) if (kvColumns[i][1] === "metadata") hasKvMetadata = true;
|
|
3579
|
+
if (!hasKvMetadata) __sbSql(s, "ALTER TABLE kv ADD COLUMN metadata TEXT", []);
|
|
3341
3580
|
// #56 — no `body` column: an object's bytes live in their own file under
|
|
3342
3581
|
// "<data-dir>/r2-blobs/" (sb_r2_put_c/sb_r2_get_c), never in the sqlite row.
|
|
3343
3582
|
__sbSql(
|
|
@@ -3436,11 +3675,12 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3436
3675
|
__sbEnsureSchema();
|
|
3437
3676
|
const store = __sbStore();
|
|
3438
3677
|
const op = msg.op;
|
|
3678
|
+
if (__sbQuotaOps[op]) __sbQuotaCheck(__sbQuotaOps[op]);
|
|
3439
3679
|
|
|
3440
3680
|
if (op === "ping") return { ok: true, op: "pong", echo: msg.msg };
|
|
3441
3681
|
|
|
3442
3682
|
if (op === "kv.get") {
|
|
3443
|
-
const r = __sbSql(store, "SELECT value, expires_at, binary FROM kv WHERE ns = ? AND key = ?", [
|
|
3683
|
+
const r = __sbSql(store, "SELECT value, expires_at, binary, metadata FROM kv WHERE ns = ? AND key = ?", [
|
|
3444
3684
|
msg.ns,
|
|
3445
3685
|
msg.key,
|
|
3446
3686
|
]);
|
|
@@ -3450,7 +3690,14 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3450
3690
|
__sbSql(store, "DELETE FROM kv WHERE ns = ? AND key = ?", [msg.ns, msg.key]);
|
|
3451
3691
|
return { ok: true, found: false, value: null };
|
|
3452
3692
|
}
|
|
3453
|
-
return {
|
|
3693
|
+
return {
|
|
3694
|
+
ok: true,
|
|
3695
|
+
found: true,
|
|
3696
|
+
value: r.rows[0][0],
|
|
3697
|
+
binary: r.rows[0][2] === 1,
|
|
3698
|
+
metadata: r.rows[0][3],
|
|
3699
|
+
expiration: expiresAt == null ? null : Math.floor(expiresAt / 1000),
|
|
3700
|
+
};
|
|
3454
3701
|
}
|
|
3455
3702
|
if (op === "kv.put") {
|
|
3456
3703
|
// #58 — CF rejects expirationTtl under 60 seconds; matched here.
|
|
@@ -3464,8 +3711,8 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3464
3711
|
}
|
|
3465
3712
|
__sbSql(
|
|
3466
3713
|
store,
|
|
3467
|
-
"INSERT INTO kv (ns, key, value, expires_at, binary) VALUES (?1,?2,?3,?4,?5) ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5",
|
|
3468
|
-
[msg.ns, msg.key, msg.value, expiresAt, msg.binary === true ? 1 : 0],
|
|
3714
|
+
"INSERT INTO kv (ns, key, value, expires_at, binary, metadata) VALUES (?1,?2,?3,?4,?5,?6) ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4, binary = ?5, metadata = ?6",
|
|
3715
|
+
[msg.ns, msg.key, msg.value, expiresAt, msg.binary === true ? 1 : 0, msg.metadata == null ? null : String(msg.metadata)],
|
|
3469
3716
|
);
|
|
3470
3717
|
return { ok: true };
|
|
3471
3718
|
}
|
|
@@ -3474,14 +3721,28 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3474
3721
|
return { ok: true };
|
|
3475
3722
|
}
|
|
3476
3723
|
if (op === "kv.list") {
|
|
3724
|
+
// The prefix matches exactly: LIKE ignored case and read "_" and "%" in it as wildcards.
|
|
3725
|
+
// With a limit (the shim from 2026-10-25), a page of entries after the cursor (the last
|
|
3726
|
+
// key of the previous page), with each key's expiration and metadata; without one,
|
|
3727
|
+
// every key, as before.
|
|
3728
|
+
const prefix = msg.prefix || "";
|
|
3729
|
+
const paged = msg.limit != null;
|
|
3730
|
+
const limit = paged ? Math.max(0, Math.floor(Number(msg.limit))) : -1;
|
|
3477
3731
|
const r = __sbSql(
|
|
3478
3732
|
store,
|
|
3479
|
-
"SELECT key FROM kv WHERE ns = ? AND key
|
|
3480
|
-
|
|
3733
|
+
"SELECT key, expires_at, metadata FROM kv WHERE ns = ?1 AND substr(key, 1, length(?2)) = ?2 AND key > ?3 " +
|
|
3734
|
+
"AND (expires_at IS NULL OR expires_at > ?4) ORDER BY key LIMIT ?5",
|
|
3735
|
+
[msg.ns, prefix, paged && msg.cursor ? String(msg.cursor) : "", Date.now(), paged ? limit + 1 : -1],
|
|
3481
3736
|
);
|
|
3482
3737
|
const keys = [];
|
|
3483
|
-
|
|
3484
|
-
|
|
3738
|
+
const entries = [];
|
|
3739
|
+
const count = paged ? Math.min(r.rows.length, limit) : r.rows.length;
|
|
3740
|
+
for (let i = 0; i < count; i++) {
|
|
3741
|
+
keys.push(r.rows[i][0]);
|
|
3742
|
+
entries.push({ name: r.rows[i][0], expiration: r.rows[i][1] == null ? null : Math.floor(r.rows[i][1] / 1000), metadata: r.rows[i][2] });
|
|
3743
|
+
}
|
|
3744
|
+
const complete = !paged || r.rows.length <= limit;
|
|
3745
|
+
return { ok: true, keys, entries, list_complete: complete, cursor: complete ? "" : keys[keys.length - 1] };
|
|
3485
3746
|
}
|
|
3486
3747
|
|
|
3487
3748
|
if (op === "assets.get") {
|
|
@@ -3586,6 +3847,7 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3586
3847
|
// exec() takes a script; a single prepare would run only its first
|
|
3587
3848
|
// statement and quietly drop the rest.
|
|
3588
3849
|
__sbD1Open(path, msg.strict === true);
|
|
3850
|
+
__sbD1Quota(path);
|
|
3589
3851
|
if (msg.strict === true && Array.isArray(msg.lines)) {
|
|
3590
3852
|
__sbD1ExecLines(path, msg.lines);
|
|
3591
3853
|
return { ok: true };
|
|
@@ -3596,6 +3858,7 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
3596
3858
|
}
|
|
3597
3859
|
const strict = msg.strict === true;
|
|
3598
3860
|
__sbD1Open(path, strict);
|
|
3861
|
+
__sbD1Quota(path);
|
|
3599
3862
|
if (op === "d1.batch") return { ok: true, results: __sbD1Batch(path, msg.statements || [], strict) };
|
|
3600
3863
|
const one = __sbD1Run(path, msg.sql, msg.params, strict);
|
|
3601
3864
|
return { ok: true, results: one.results, columns: one.columns, rows: one.rows, meta: one.meta, success: true };
|
|
@@ -4166,6 +4429,7 @@ function __sbD1ExecLines(path, lines) {
|
|
|
4166
4429
|
* itself does not know which backend it is on.
|
|
4167
4430
|
*/
|
|
4168
4431
|
globalThis.__sbR2Put = function (bucket, key, body, httpMetadata, customMetadata) {
|
|
4432
|
+
__sbQuotaCheck("R2 put()");
|
|
4169
4433
|
__sbEnsureSchema();
|
|
4170
4434
|
const reply = JSON.parse(
|
|
4171
4435
|
__sbR2PutRaw(
|
|
@@ -4195,6 +4459,7 @@ globalThis.__sbR2Get = function (bucket, key) {
|
|
|
4195
4459
|
};
|
|
4196
4460
|
|
|
4197
4461
|
globalThis.__sbR2MultipartPut = function (bucket, key, uploadId, partNumber, body) {
|
|
4462
|
+
__sbQuotaCheck("R2 uploadPart()");
|
|
4198
4463
|
__sbEnsureSchema();
|
|
4199
4464
|
const store = __sbStore();
|
|
4200
4465
|
const upload = __sbSql(
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// LTX chain restore, the same independent checker as native-ltx.ts (copied, so that test
|
|
2
|
+
// stays as it is): every file's CRCs, checksums and TXIDs verified, pages applied in order.
|
|
3
|
+
import assert from "node:assert/strict";
|
|
4
|
+
import { readFile, readdir, writeFile } from "node:fs/promises";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
// Independent CRC baseline in Bun. This intentionally uses BigInt, unlike C.
|
|
8
|
+
const table: bigint[] = [];
|
|
9
|
+
for (let i = 0; i < 256; i++) {
|
|
10
|
+
let c = BigInt(i);
|
|
11
|
+
for (let j = 0; j < 8; j++) c = (c >> 1n) ^ (c & 1n ? 0xd800000000000000n : 0n);
|
|
12
|
+
table.push(c);
|
|
13
|
+
}
|
|
14
|
+
function crc(data: Uint8Array, init = 0xffffffffffffffffn) {
|
|
15
|
+
let c = init;
|
|
16
|
+
for (const b of data) c = table[Number((c ^ BigInt(b)) & 255n)] ^ (c >> 8n);
|
|
17
|
+
return c;
|
|
18
|
+
}
|
|
19
|
+
const flag = 0x8000000000000000n,
|
|
20
|
+
mask = 0xffffffffffffffffn;
|
|
21
|
+
function pageCrc(pgno: number, page: Uint8Array) {
|
|
22
|
+
const p = new Uint8Array(4);
|
|
23
|
+
new DataView(p.buffer).setUint32(0, pgno);
|
|
24
|
+
return flag | (crc(page, crc(p)) ^ mask);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function restore(epoch: string, output: string) {
|
|
28
|
+
const files = (await readdir(epoch)).filter((n) => n.endsWith(".ltx")).sort();
|
|
29
|
+
let db = new Uint8Array(0),
|
|
30
|
+
checksum = 0n,
|
|
31
|
+
txid = 0n;
|
|
32
|
+
for (const file of files) {
|
|
33
|
+
const b = new Uint8Array(await readFile(join(epoch, file)));
|
|
34
|
+
const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
|
|
35
|
+
assert.equal(new TextDecoder().decode(b.subarray(0, 4)), "LTX1");
|
|
36
|
+
const pageSize = v.getUint32(8),
|
|
37
|
+
commit = v.getUint32(12);
|
|
38
|
+
assert.equal(v.getBigUint64(16), txid + 1n);
|
|
39
|
+
assert.equal(v.getBigUint64(40), checksum);
|
|
40
|
+
const next = new Uint8Array(commit * pageSize);
|
|
41
|
+
next.set(db.subarray(0, next.length));
|
|
42
|
+
db = next;
|
|
43
|
+
let offset = 100,
|
|
44
|
+
hash = crc(b.subarray(0, 100));
|
|
45
|
+
const entries = [];
|
|
46
|
+
for (;;) {
|
|
47
|
+
const begin = offset,
|
|
48
|
+
pgno = v.getUint32(offset),
|
|
49
|
+
flags = v.getUint16(offset + 4);
|
|
50
|
+
hash = crc(b.subarray(offset, offset + 6), hash);
|
|
51
|
+
offset += 6;
|
|
52
|
+
if (!pgno) {
|
|
53
|
+
assert.equal(flags, 0);
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
assert.equal(flags, 1);
|
|
57
|
+
assert(pgno <= commit);
|
|
58
|
+
const size = v.getUint32(offset);
|
|
59
|
+
hash = crc(b.subarray(offset, offset + 4), hash);
|
|
60
|
+
offset += 4;
|
|
61
|
+
const stop = offset + size;
|
|
62
|
+
assert(stop <= b.length);
|
|
63
|
+
assert.equal(b[offset++], 0xf0);
|
|
64
|
+
let literals = 15,
|
|
65
|
+
byte;
|
|
66
|
+
do {
|
|
67
|
+
byte = b[offset++];
|
|
68
|
+
literals += byte;
|
|
69
|
+
} while (byte === 255);
|
|
70
|
+
assert.equal(literals, pageSize);
|
|
71
|
+
assert.equal(offset + pageSize, stop);
|
|
72
|
+
const page = b.subarray(offset, stop);
|
|
73
|
+
db.set(page, (pgno - 1) * pageSize);
|
|
74
|
+
hash = crc(page, hash);
|
|
75
|
+
offset = stop;
|
|
76
|
+
entries.push({ pgno, offset: begin, size: stop - begin });
|
|
77
|
+
}
|
|
78
|
+
const indexStart = offset;
|
|
79
|
+
function varint() {
|
|
80
|
+
let n = 0n,
|
|
81
|
+
s = 0n;
|
|
82
|
+
for (let i = 0; i < 10; i++) {
|
|
83
|
+
assert(offset < b.length);
|
|
84
|
+
const c = b[offset++];
|
|
85
|
+
n |= BigInt(c & 127) << s;
|
|
86
|
+
if (c < 128) return Number(n);
|
|
87
|
+
s += 7n;
|
|
88
|
+
}
|
|
89
|
+
throw new Error("varint overflow");
|
|
90
|
+
}
|
|
91
|
+
for (const entry of entries) {
|
|
92
|
+
assert.equal(varint(), entry.pgno);
|
|
93
|
+
assert.equal(varint(), entry.offset);
|
|
94
|
+
assert.equal(varint(), entry.size);
|
|
95
|
+
}
|
|
96
|
+
assert.equal(varint(), 0);
|
|
97
|
+
assert.equal(v.getBigUint64(offset), BigInt(offset - indexStart));
|
|
98
|
+
offset += 8;
|
|
99
|
+
hash = crc(b.subarray(indexStart, offset + 8), hash);
|
|
100
|
+
checksum = flag;
|
|
101
|
+
for (let p = 1; p <= commit; p++)
|
|
102
|
+
checksum = flag | (checksum ^ pageCrc(p, db.subarray((p - 1) * pageSize, p * pageSize)));
|
|
103
|
+
assert.equal(v.getBigUint64(offset), checksum, file);
|
|
104
|
+
assert.equal(v.getBigUint64(offset + 8), flag | (hash ^ mask), file);
|
|
105
|
+
assert.equal(offset + 16, b.length);
|
|
106
|
+
txid = v.getBigUint64(24);
|
|
107
|
+
}
|
|
108
|
+
await writeFile(output, db);
|
|
109
|
+
return { files: files.length, txid };
|
|
110
|
+
}
|