@sproutboat/runtime 0.44.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/package.json +1 -1
- package/src/asset-rules.test.js +51 -30
- package/src/native-fetch-prelude.js +1213 -67
- package/src/r2-lazy.test.js +101 -0
- package/src/transport-broker.js +13 -0
- package/src/transport-embedded.js +534 -47
- package/test/fixtures/asset-redirect-cases.ts +28 -0
- package/test/fixtures/do-evict-app.js +172 -0
- package/test/native-asset-redirects.ts +80 -0
- package/test/native-cc.ts +178 -0
- package/test/native-compile.ts +53 -22
- package/test/native-db-table.ts +244 -0
- package/test/native-do-evict.ts +345 -0
- package/test/native-lto.ts +79 -0
|
@@ -27,6 +27,7 @@ Porffor.c`
|
|
|
27
27
|
#include <string.h>
|
|
28
28
|
#include <strings.h>
|
|
29
29
|
#include <sys/time.h>
|
|
30
|
+
#include <time.h>
|
|
30
31
|
#include <pthread.h>
|
|
31
32
|
#include <bearssl.h>
|
|
32
33
|
|
|
@@ -65,6 +66,11 @@ extern int sqlite3_extended_errcode(sqlite3*);
|
|
|
65
66
|
extern int sqlite3_error_offset(sqlite3*);
|
|
66
67
|
extern const char* sqlite3_sql(sqlite3_stmt*);
|
|
67
68
|
extern int sqlite3_total_changes(sqlite3*);
|
|
69
|
+
extern int sqlite3_get_autocommit(sqlite3*);
|
|
70
|
+
extern int sqlite3_db_status(sqlite3*, int, int*, int*, int);
|
|
71
|
+
extern int sqlite3_db_release_memory(sqlite3*);
|
|
72
|
+
extern long long sqlite3_soft_heap_limit64(long long);
|
|
73
|
+
extern long long sqlite3_memory_used(void);
|
|
68
74
|
|
|
69
75
|
// baronunread/sproutboat#176 — defined in render.js, alongside
|
|
70
76
|
// porf_native_fetch_read_value (which every inline-C block below already
|
|
@@ -79,7 +85,7 @@ extern int porf_native_fetch_read_raw_bytes(jsval value, const char** out_buf, s
|
|
|
79
85
|
// SQLITE_STATIC: "this buffer is yours to read and outlives the statement".
|
|
80
86
|
// Worth using for an object body — the alternative is sqlite copying it.
|
|
81
87
|
#define SB_SQLITE_STATIC ((void*)0)
|
|
82
|
-
#define SB_MAX_DB
|
|
88
|
+
#define SB_MAX_DB 64
|
|
83
89
|
|
|
84
90
|
static sqlite3* sb_dbs[SB_MAX_DB];
|
|
85
91
|
static char sb_db_names[SB_MAX_DB][256];
|
|
@@ -1173,8 +1179,9 @@ static int sb_ltx_opted(const char *path) {
|
|
|
1173
1179
|
}
|
|
1174
1180
|
// END LTX NATIVE
|
|
1175
1181
|
|
|
1176
|
-
// One handle per path,
|
|
1177
|
-
//
|
|
1182
|
+
// One handle per path, kept open until the slot is needed (Durable Object SQL) or for the process lifetime
|
|
1183
|
+
// (everything else). Returns an index, or -1.
|
|
1184
|
+
// BEGIN DB TABLE NATIVE: also compiled directly by the handle-table regression (packages/runtime/test/native-db-table.ts).
|
|
1178
1185
|
// Create every parent directory of path, like mkdir -p. A standalone binary
|
|
1179
1186
|
// has no launcher to prepare its data directory, and sqlite3_open creates files
|
|
1180
1187
|
// but never the directories above them.
|
|
@@ -1191,13 +1198,31 @@ static void sb_mkdirs(const char* path) {
|
|
|
1191
1198
|
}
|
|
1192
1199
|
}
|
|
1193
1200
|
|
|
1194
|
-
// Durable Objects SQL (storage.sql): one database per object, under "/do-sql/". At most
|
|
1195
|
-
//
|
|
1196
|
-
|
|
1201
|
+
// Durable Objects SQL (storage.sql): one database per object, under "/do-sql/". At most sb_do_sql_open of them
|
|
1202
|
+
// stay open (8 until the memory limit is known, then clamp(T / 160 KiB, 8, 48); 48 with no limit); the one used
|
|
1203
|
+
// longest ago, and not inside a transaction, is closed for the next. A Durable Object instance leaving memory closes
|
|
1204
|
+
// its handle at once (op do.sql.close).
|
|
1205
|
+
static int sb_do_sql_open = 8;
|
|
1197
1206
|
static unsigned long sb_db_used[SB_MAX_DB];
|
|
1198
1207
|
static unsigned long sb_db_tick = 0;
|
|
1208
|
+
static int sb_db_last = -1;
|
|
1199
1209
|
static int sb_is_do_sql(const char* path) { return strstr(path, "/do-sql/") != 0; }
|
|
1200
|
-
static
|
|
1210
|
+
static int sb_db_close(int idx);
|
|
1211
|
+
|
|
1212
|
+
// SQLite page caches (plan 016 section 5 as plan 018 phase 1 delivers it). Every database starts at 256 KiB. T is the
|
|
1213
|
+
// target for all of them together: SB_MEMORY_LIMIT / 8 (4 MiB at 32 MiB), none without a limit. Each database may
|
|
1214
|
+
// grow, on evidence, to min(2 MiB, T / open databases); when a new one opens and the sizes no longer fit under T the
|
|
1215
|
+
// stalest ones go back to their share. SB_SQLITE_CACHE_MB, when set, is the old fixed size and turns all of this off.
|
|
1216
|
+
#define SB_CACHE_START_KB 256
|
|
1217
|
+
#define SB_CACHE_MIN_KB 64
|
|
1218
|
+
#define SB_CACHE_MAX_KB 2048
|
|
1219
|
+
static long sb_cache_target_kb = 0;
|
|
1220
|
+
static int sb_cache_fixed_kb = 0;
|
|
1221
|
+
static int sb_cache_kb[SB_MAX_DB];
|
|
1222
|
+
static unsigned long sb_db_calls[SB_MAX_DB];
|
|
1223
|
+
static long sb_db_changes0[SB_MAX_DB];
|
|
1224
|
+
static unsigned long sb_stat_closes, sb_stat_busy, sb_stat_ckpt_fail, sb_stat_grow, sb_stat_shrink, sb_stat_opens;
|
|
1225
|
+
static long sb_stat_open_us_last, sb_stat_open_us_max;
|
|
1201
1226
|
|
|
1202
1227
|
// SQLite allocates 20 pages in one block for every page cache it creates, and a statement that
|
|
1203
1228
|
// builds a temporary table (INSERT ... RETURNING, a sort) creates one: an 88 KiB mmap and munmap
|
|
@@ -1209,33 +1234,113 @@ static void sb_sqlite_boot(void) {
|
|
|
1209
1234
|
if (sb_sqlite_booted) return;
|
|
1210
1235
|
sb_sqlite_booted = 1;
|
|
1211
1236
|
sqlite3_config(7 /* SQLITE_CONFIG_PAGECACHE */, (void*)0, 0, 0);
|
|
1237
|
+
const char* mb = getenv("SB_SQLITE_CACHE_MB");
|
|
1238
|
+
long mib = mb && *mb ? strtol(mb, 0, 10) : 0;
|
|
1239
|
+
if (mib >= 1 && mib <= 4096) sb_cache_fixed_kb = (int)(mib * 1024);
|
|
1240
|
+
const char* limit = getenv("SB_MEMORY_LIMIT");
|
|
1241
|
+
unsigned long long bytes = limit && *limit ? strtoull(limit, 0, 10) : 0;
|
|
1242
|
+
if (bytes >= 8ull << 20) {
|
|
1243
|
+
sb_cache_target_kb = (long)(bytes / 8 / 1024);
|
|
1244
|
+
long slots = sb_cache_target_kb / 160;
|
|
1245
|
+
sb_do_sql_open = slots < 8 ? 8 : slots > 48 ? 48 : (int)slots;
|
|
1246
|
+
// SQLite frees cache pages of every database before it goes over T; one statement that needs more still succeeds.
|
|
1247
|
+
sqlite3_soft_heap_limit64((long long)(bytes / 8));
|
|
1248
|
+
} else sb_do_sql_open = 48;
|
|
1249
|
+
}
|
|
1250
|
+
static int sb_open_dbs(void) {
|
|
1251
|
+
int n = 0;
|
|
1252
|
+
for (int i = 0; i < sb_db_count; i++) if (sb_dbs[i]) n++;
|
|
1253
|
+
return n;
|
|
1254
|
+
}
|
|
1255
|
+
// The most one database may hold now: min(2 MiB, T / open databases), and never under 64 KiB.
|
|
1256
|
+
static int sb_cache_share_kb(void) {
|
|
1257
|
+
if (!sb_cache_target_kb) return SB_CACHE_MAX_KB;
|
|
1258
|
+
int open = sb_open_dbs();
|
|
1259
|
+
long share = sb_cache_target_kb / (open ? open : 1);
|
|
1260
|
+
return share < SB_CACHE_MIN_KB ? SB_CACHE_MIN_KB : share > SB_CACHE_MAX_KB ? SB_CACHE_MAX_KB : (int)share;
|
|
1261
|
+
}
|
|
1262
|
+
static void sb_cache_set(int idx, int kb) {
|
|
1263
|
+
char pragma[48];
|
|
1264
|
+
snprintf(pragma, sizeof(pragma), "PRAGMA cache_size=-%d", kb);
|
|
1265
|
+
sqlite3_exec(sb_dbs[idx], pragma, 0, 0, 0);
|
|
1266
|
+
sb_cache_kb[idx] = kb;
|
|
1267
|
+
}
|
|
1268
|
+
// After a database opens: while the sizes together are over T, the stalest database above its share goes back to it.
|
|
1269
|
+
static void sb_cache_rebalance(int keep) {
|
|
1270
|
+
if (!sb_cache_target_kb || sb_cache_fixed_kb) return;
|
|
1271
|
+
for (;;) {
|
|
1272
|
+
long sum = 0;
|
|
1273
|
+
int share = sb_cache_share_kb(), victim = -1;
|
|
1274
|
+
for (int i = 0; i < sb_db_count; i++) {
|
|
1275
|
+
if (!sb_dbs[i]) continue;
|
|
1276
|
+
sum += sb_cache_kb[i];
|
|
1277
|
+
if (i != keep && sb_cache_kb[i] > share && (victim < 0 || sb_db_used[i] < sb_db_used[victim])) victim = i;
|
|
1278
|
+
}
|
|
1279
|
+
if (sum <= sb_cache_target_kb || victim < 0) return;
|
|
1280
|
+
sb_cache_set(victim, share);
|
|
1281
|
+
sqlite3_exec(sb_dbs[victim], "PRAGMA shrink_memory", 0, 0, 0);
|
|
1282
|
+
sb_stat_shrink++;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
// Every 64th use of a database: over 90% full and over 10% misses since the last look doubles its cache, up to its share.
|
|
1286
|
+
static void sb_cache_tune(int idx) {
|
|
1287
|
+
if (sb_cache_fixed_kb || sb_cache_kb[idx] >= SB_CACHE_MAX_KB) return;
|
|
1288
|
+
int used = 0, hit = 0, miss = 0, high = 0;
|
|
1289
|
+
sqlite3_db_status(sb_dbs[idx], 1 /* CACHE_USED */, &used, &high, 0);
|
|
1290
|
+
sqlite3_db_status(sb_dbs[idx], 7 /* CACHE_HIT */, &hit, &high, 1);
|
|
1291
|
+
sqlite3_db_status(sb_dbs[idx], 8 /* CACHE_MISS */, &miss, &high, 1);
|
|
1292
|
+
if ((long long)used * 10 < (long long)sb_cache_kb[idx] * 1024 * 9 || (long long)miss * 10 <= (long long)(hit + miss)) return;
|
|
1293
|
+
int share = sb_cache_share_kb(), next = sb_cache_kb[idx] * 2;
|
|
1294
|
+
if (next > share) next = share;
|
|
1295
|
+
if (next <= sb_cache_kb[idx]) return;
|
|
1296
|
+
sb_cache_set(idx, next);
|
|
1297
|
+
sb_stat_grow++;
|
|
1298
|
+
sb_cache_rebalance(idx);
|
|
1299
|
+
}
|
|
1300
|
+
static void sb_db_touch(int i) {
|
|
1301
|
+
sb_db_used[i] = ++sb_db_tick;
|
|
1302
|
+
sb_db_last = i;
|
|
1303
|
+
if ((++sb_db_calls[i] & 63) == 0) sb_cache_tune(i);
|
|
1304
|
+
}
|
|
1305
|
+
static long sb_now_us(void) {
|
|
1306
|
+
struct timespec t;
|
|
1307
|
+
clock_gettime(CLOCK_MONOTONIC, &t);
|
|
1308
|
+
return (long)t.tv_sec * 1000000L + t.tv_nsec / 1000;
|
|
1212
1309
|
}
|
|
1213
1310
|
static int sb_db_for(const char* path) {
|
|
1311
|
+
if (sb_db_last >= 0 && sb_dbs[sb_db_last] && strcmp(sb_db_names[sb_db_last], path) == 0) {
|
|
1312
|
+
sb_db_touch(sb_db_last);
|
|
1313
|
+
return sb_ltx_states[sb_db_last].failed ? -1 : sb_db_last;
|
|
1314
|
+
}
|
|
1214
1315
|
for (int i = 0; i < sb_db_count; i++) {
|
|
1215
1316
|
if (sb_dbs[i] && strcmp(sb_db_names[i], path) == 0) {
|
|
1216
|
-
|
|
1317
|
+
sb_db_touch(i);
|
|
1217
1318
|
return sb_ltx_states[i].failed ? -1 : i;
|
|
1218
1319
|
}
|
|
1219
1320
|
}
|
|
1220
1321
|
sb_sqlite_boot();
|
|
1322
|
+
const long t0 = sb_now_us();
|
|
1221
1323
|
const int do_sql = sb_is_do_sql(path);
|
|
1222
1324
|
// A name that would not fit could never be found again.
|
|
1223
1325
|
if (do_sql && strlen(path) >= sizeof(sb_db_names[0])) return -1;
|
|
1224
1326
|
int replicated = do_sql ? 0 : sb_ltx_opted(path), lock = -1;
|
|
1225
1327
|
if (replicated < 0 || (replicated && strlen(path) >= sizeof(sb_db_names[0]))) return -1;
|
|
1226
|
-
// A Durable Object SQL database
|
|
1227
|
-
//
|
|
1228
|
-
int slot =
|
|
1328
|
+
// A closed slot is reused first. A Durable Object SQL database takes the slot of the one used longest ago
|
|
1329
|
+
// (outside a transaction) once sb_do_sql_open are open; any other database does the same when every slot is in use.
|
|
1330
|
+
int slot = -1, lru = -1, open_do_sql = 0;
|
|
1229
1331
|
for (int i = 0; i < sb_db_count; i++) {
|
|
1230
|
-
if (!sb_dbs[i]
|
|
1332
|
+
if (!sb_dbs[i]) { if (slot < 0) slot = i; continue; }
|
|
1333
|
+
if (!sb_is_do_sql(sb_db_names[i])) continue;
|
|
1231
1334
|
open_do_sql++;
|
|
1232
|
-
if (lru < 0 || sb_db_used[i] < sb_db_used[lru]) lru = i;
|
|
1335
|
+
if (sqlite3_get_autocommit(sb_dbs[i]) && (lru < 0 || sb_db_used[i] < sb_db_used[lru])) lru = i;
|
|
1233
1336
|
}
|
|
1234
|
-
if (
|
|
1235
|
-
|
|
1337
|
+
if (slot < 0 && sb_db_count < SB_MAX_DB) slot = sb_db_count;
|
|
1338
|
+
if (!replicated && ((do_sql && open_do_sql >= sb_do_sql_open) || slot < 0)) {
|
|
1339
|
+
// Every candidate inside a transaction, or none open: the new database cannot be opened now.
|
|
1340
|
+
if (lru < 0 || sb_db_close(lru)) return -1;
|
|
1236
1341
|
slot = lru;
|
|
1237
1342
|
}
|
|
1238
|
-
if (slot
|
|
1343
|
+
if (slot < 0) return -1;
|
|
1239
1344
|
sqlite3* db = 0;
|
|
1240
1345
|
sb_mkdirs(path);
|
|
1241
1346
|
if (replicated) {
|
|
@@ -1266,16 +1371,18 @@ static int sb_db_for(const char* path) {
|
|
|
1266
1371
|
// The transport itself only reads the variable, so a host that embeds it (sproutd, the broker) stays
|
|
1267
1372
|
// in shared mode unless it is set. A standalone binary sets it to 1 at startup (patch.ts, the shim's
|
|
1268
1373
|
// constructor) unless the operator has set it, so SB_SQLITE_EXCLUSIVE=0 opts a standalone binary out.
|
|
1269
|
-
//
|
|
1374
|
+
// The page cache: see the sizing comment above; SB_SQLITE_CACHE_MB is a fixed size in MiB.
|
|
1270
1375
|
{
|
|
1271
1376
|
const char* excl = getenv("SB_SQLITE_EXCLUSIVE");
|
|
1272
1377
|
if (excl && excl[0] == '1' && !excl[1] && !replicated) sqlite3_exec(db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0);
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1378
|
+
}
|
|
1379
|
+
sb_dbs[slot] = db;
|
|
1380
|
+
snprintf(sb_db_names[slot], 256, "%s", path);
|
|
1381
|
+
if (slot >= sb_db_count) sb_db_count = slot + 1; // before the sizing below, which counts open databases
|
|
1382
|
+
{
|
|
1383
|
+
const int share = sb_cache_share_kb();
|
|
1384
|
+
const int kb = sb_cache_fixed_kb ? sb_cache_fixed_kb : (SB_CACHE_START_KB < share ? SB_CACHE_START_KB : share);
|
|
1385
|
+
sb_cache_set(slot, kb);
|
|
1279
1386
|
}
|
|
1280
1387
|
sqlite3_exec(db,
|
|
1281
1388
|
"PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA busy_timeout=5000;"
|
|
@@ -1283,9 +1390,10 @@ static int sb_db_for(const char* path) {
|
|
|
1283
1390
|
0, 0, 0);
|
|
1284
1391
|
// Durable Object SQLite enforces foreign keys, as in workerd.
|
|
1285
1392
|
if (do_sql) sqlite3_exec(db, "PRAGMA foreign_keys=ON", 0, 0, 0);
|
|
1393
|
+
sb_db_changes0[slot] = sqlite3_total_changes(db);
|
|
1394
|
+
sb_db_calls[slot] = 0;
|
|
1286
1395
|
sb_db_used[slot] = ++sb_db_tick;
|
|
1287
|
-
|
|
1288
|
-
snprintf(sb_db_names[slot], 256, "%s", path);
|
|
1396
|
+
sb_db_last = slot;
|
|
1289
1397
|
if (replicated && sb_ltx_start(slot, db, path, lock)) {
|
|
1290
1398
|
sb_ltx_states[slot].failed = 1;
|
|
1291
1399
|
fprintf(stderr, "LTX initialization failed for %s\n", path);
|
|
@@ -1293,10 +1401,12 @@ static int sb_db_for(const char* path) {
|
|
|
1293
1401
|
if (sb_ltx_states[slot].read_fd >= 0) close(sb_ltx_states[slot].read_fd);
|
|
1294
1402
|
sb_ltx_states[slot].read_fd = -1;
|
|
1295
1403
|
sb_dbs[slot] = 0;
|
|
1296
|
-
if (slot == sb_db_count) sb_db_count++;
|
|
1297
1404
|
close(lock); return -1;
|
|
1298
1405
|
}
|
|
1299
|
-
|
|
1406
|
+
sb_stat_opens++;
|
|
1407
|
+
sb_cache_rebalance(slot);
|
|
1408
|
+
sb_stat_open_us_last = sb_now_us() - t0;
|
|
1409
|
+
if (sb_stat_open_us_last > sb_stat_open_us_max) sb_stat_open_us_max = sb_stat_open_us_last;
|
|
1300
1410
|
return slot;
|
|
1301
1411
|
}
|
|
1302
1412
|
|
|
@@ -1308,7 +1418,7 @@ static int sb_db_for(const char* path) {
|
|
|
1308
1418
|
// ponytail: FIFO eviction over 32 slots, linear scan, no lock (the runtime
|
|
1309
1419
|
// serves one turn at a time). Move to LRU + a hash if a real app keeps >32 hot
|
|
1310
1420
|
// statements per db, or if execution ever overlaps. Entries are never finalized
|
|
1311
|
-
// on shutdown
|
|
1421
|
+
// on shutdown; sb_db_close finalizes a database's entries when its handle goes.
|
|
1312
1422
|
#define SB_STMT_CACHE 32
|
|
1313
1423
|
typedef struct { char* sql; sqlite3_stmt* st; } sb_cached_stmt;
|
|
1314
1424
|
static sb_cached_stmt sb_stmt_cache[SB_MAX_DB][SB_STMT_CACHE];
|
|
@@ -1336,9 +1446,18 @@ static sqlite3_stmt* sb_stmt_get(int idx, sqlite3* db, const char* sql) {
|
|
|
1336
1446
|
return st;
|
|
1337
1447
|
}
|
|
1338
1448
|
|
|
1339
|
-
//
|
|
1449
|
+
// The one place a handle is closed: the slot LRU, a Durable Object instance leaving memory (do.sql.close), and later
|
|
1450
|
+
// a capturing handle (plan 013 phase D extends it at the two marked steps and removes the LRU's capture guard; it must not
|
|
1451
|
+
// add a second close path). The order is a contract:
|
|
1340
1452
|
static unsigned char sb_quota_frozen[SB_MAX_DB];
|
|
1341
|
-
|
|
1453
|
+
// Returns 0 when the handle is closed (or was not open), -1 when it is left open: a statement outlives its call
|
|
1454
|
+
// (SQLITE_BUSY: counted, never sqlite3_close_v2), or the connection is inside a transaction.
|
|
1455
|
+
static int sb_db_close(int idx) {
|
|
1456
|
+
sqlite3* db = sb_dbs[idx];
|
|
1457
|
+
if (!db) return 0;
|
|
1458
|
+
// A savepoint or transaction held across calls: closing would roll it back silently.
|
|
1459
|
+
if (!sqlite3_get_autocommit(db)) return -1;
|
|
1460
|
+
// 1. finalize the slot's cached statements;
|
|
1342
1461
|
for (int i = 0; i < SB_STMT_CACHE; i++) {
|
|
1343
1462
|
if (sb_stmt_cache[idx][i].st) sqlite3_finalize(sb_stmt_cache[idx][i].st);
|
|
1344
1463
|
free(sb_stmt_cache[idx][i].sql);
|
|
@@ -1346,11 +1465,61 @@ static void sb_db_evict(int idx) {
|
|
|
1346
1465
|
sb_stmt_cache[idx][i].sql = 0;
|
|
1347
1466
|
}
|
|
1348
1467
|
sb_stmt_fifo[idx] = 0;
|
|
1349
|
-
|
|
1468
|
+
// 2. (phase D inserts: unhook capture, write what is pending)
|
|
1469
|
+
// 3. if the connection wrote since it was opened, a TRUNCATE checkpoint and its result. With synchronous=NORMAL this is
|
|
1470
|
+
// the fsync that makes the data durable on the local disk. A read-only open skips it; sqlite3_close below still
|
|
1471
|
+
// checkpoints a WAL that holds schema changes (they do not count as changes), as it does for the last connection.
|
|
1472
|
+
if (sqlite3_total_changes(db) != sb_db_changes0[idx]) {
|
|
1473
|
+
int frames = 0, done = 0;
|
|
1474
|
+
if (sqlite3_wal_checkpoint_v2(db, "main", 3 /* TRUNCATE */, &frames, &done)) sb_stat_ckpt_fail++;
|
|
1475
|
+
}
|
|
1476
|
+
// 4. (phase D inserts: the clean position record)
|
|
1477
|
+
// 5. sqlite3_close must return SQLITE_OK. SQLITE_BUSY means a leaked statement: the slot stays open.
|
|
1478
|
+
if (sqlite3_close(db)) {
|
|
1479
|
+
sb_stat_busy++;
|
|
1480
|
+
return -1;
|
|
1481
|
+
}
|
|
1482
|
+
// 6. only then close any raw descriptor on the same inode (capture's read_fd and lock): fcntl locks belong to the
|
|
1483
|
+
// process, so closing a descriptor of the file while the SQLite handle is open would drop SQLite's locks. Object
|
|
1484
|
+
// SQL has none; the order is written so that phase D cannot get it wrong.
|
|
1485
|
+
// 7. clear the slot.
|
|
1486
|
+
if (sb_ltx_states[idx].db) memset(&sb_ltx_states[idx], 0, sizeof(sb_ltx_states[idx]));
|
|
1350
1487
|
sb_dbs[idx] = 0;
|
|
1351
1488
|
sb_db_names[idx][0] = 0;
|
|
1352
1489
|
sb_quota_frozen[idx] = 0;
|
|
1490
|
+
sb_cache_kb[idx] = 0;
|
|
1491
|
+
sb_db_calls[idx] = 0;
|
|
1492
|
+
if (sb_db_last == idx) sb_db_last = -1;
|
|
1493
|
+
sb_stat_closes++;
|
|
1494
|
+
return 0;
|
|
1495
|
+
}
|
|
1496
|
+
// do.sql.close: the handle of one path. 0: not open, 1: closed, -1: left open.
|
|
1497
|
+
static int sb_db_close_path(const char* path) {
|
|
1498
|
+
for (int i = 0; i < sb_db_count; i++)
|
|
1499
|
+
if (sb_dbs[i] && strcmp(sb_db_names[i], path) == 0) return sb_db_close(i) ? -1 : 1;
|
|
1500
|
+
return 0;
|
|
1501
|
+
}
|
|
1502
|
+
// Hands SQLite's cached pages back, from every open database; returns the bytes SQLite memory fell by.
|
|
1503
|
+
static long long sb_db_release_all(void) {
|
|
1504
|
+
long long before = sqlite3_memory_used();
|
|
1505
|
+
for (int i = 0; i < sb_db_count; i++) if (sb_dbs[i]) sqlite3_db_release_memory(sb_dbs[i]);
|
|
1506
|
+
return before - sqlite3_memory_used();
|
|
1507
|
+
}
|
|
1508
|
+
// The handle table's counters as one JSON line, for the host note (plan 018) and the tests.
|
|
1509
|
+
static int sb_db_stats(char* out, size_t cap) {
|
|
1510
|
+
int open = 0, open_do = 0;
|
|
1511
|
+
long cache_kb = 0;
|
|
1512
|
+
for (int i = 0; i < sb_db_count; i++) {
|
|
1513
|
+
if (!sb_dbs[i]) continue;
|
|
1514
|
+
open++;
|
|
1515
|
+
cache_kb += sb_cache_kb[i];
|
|
1516
|
+
if (sb_is_do_sql(sb_db_names[i])) open_do++;
|
|
1517
|
+
}
|
|
1518
|
+
return snprintf(out, cap,
|
|
1519
|
+
"{\"open\":%d,\"doOpen\":%d,\"doSlots\":%d,\"cacheKb\":%ld,\"targetKb\":%ld,\"memUsed\":%lld,\"opens\":%lu,\"closes\":%lu,\"busy\":%lu,\"ckptFail\":%lu,\"grow\":%lu,\"shrink\":%lu,\"openUsLast\":%ld,\"openUsMax\":%ld}",
|
|
1520
|
+
open, open_do, sb_do_sql_open, cache_kb, sb_cache_target_kb, sqlite3_memory_used(), sb_stat_opens, sb_stat_closes, sb_stat_busy, sb_stat_ckpt_fail, sb_stat_grow, sb_stat_shrink, sb_stat_open_us_last, sb_stat_open_us_max);
|
|
1353
1521
|
}
|
|
1522
|
+
// END DB TABLE NATIVE
|
|
1354
1523
|
|
|
1355
1524
|
// --- a growable output buffer, for building the reply JSON ------------------
|
|
1356
1525
|
typedef struct { char* p; size_t len; size_t cap; } sb_buf;
|
|
@@ -1879,6 +2048,22 @@ static u32 sb_r2_get_c(const char* path, const char* bucket, const char* key, co
|
|
|
1879
2048
|
return out;
|
|
1880
2049
|
}
|
|
1881
2050
|
|
|
2051
|
+
// Reads at most CAP bytes at OFFSET of an object's file into OUT, and nothing else of it: a range read
|
|
2052
|
+
// (R2 get with a range) and each chunk of a streamed body come through here, so an object of any size is
|
|
2053
|
+
// never in memory whole. Returns the byte count (short at the end of the object), or -1 when the file
|
|
2054
|
+
// cannot be read. The file is named by the object's generation, which a put never rewrites.
|
|
2055
|
+
static long sb_r2_read_into_c(const char* path, const char* bucket, const char* key, const char* generation, size_t offset, char* out, size_t cap) {
|
|
2056
|
+
char blobpath[1024];
|
|
2057
|
+
sb_r2_blob_path(blobpath, sizeof(blobpath), path, bucket, key, generation);
|
|
2058
|
+
FILE* f = fopen(blobpath, "rb");
|
|
2059
|
+
if (!f) return -1;
|
|
2060
|
+
if (offset && fseek(f, (long)offset, SEEK_SET) != 0) { fclose(f); return -1; }
|
|
2061
|
+
size_t rd = cap ? fread(out, 1, cap, f) : 0;
|
|
2062
|
+
int bad = ferror(f);
|
|
2063
|
+
fclose(f);
|
|
2064
|
+
return bad ? -1 : (long)rd;
|
|
2065
|
+
}
|
|
2066
|
+
|
|
1882
2067
|
// Best-effort: a metadata row deleted with no blob file (or vice versa) is
|
|
1883
2068
|
// still a correct end state, so a missing file here is not an error.
|
|
1884
2069
|
static void sb_r2_delete_blob(const char* path, const char* bucket, const char* key, const char* generation) {
|
|
@@ -2119,6 +2304,95 @@ size_t sb_r2_transfer_download_read(sb_r2_transfer_ctx* ctx, size_t offset, char
|
|
|
2119
2304
|
}
|
|
2120
2305
|
void sb_r2_transfer_download_close(sb_r2_transfer_ctx* ctx) { if (ctx) { if (ctx->file) fclose(ctx->file); free(ctx); } }
|
|
2121
2306
|
|
|
2307
|
+
// A host whose workers serve transfer tickets themselves (sproutd: the frontend forwards /__sb/r2/transfer/...
|
|
2308
|
+
// like any request, with its body streamed) drives the same context from JavaScript, one step at a time.
|
|
2309
|
+
// The contexts live here, in a small table (a handle is its index plus one), so JavaScript never holds a
|
|
2310
|
+
// pointer; a handle is released by finishing, aborting or closing it. An upload step moves what the request
|
|
2311
|
+
// body has buffered (at most 1 MiB, in 64 KiB pieces from the stack) into the object's temporary file, and a
|
|
2312
|
+
// download read fills a Uint8Array from the object's file: object bytes are never strings or JSON.
|
|
2313
|
+
#define SB_XFER_MAX 16
|
|
2314
|
+
static sb_r2_transfer_ctx* sb_xfer[SB_XFER_MAX];
|
|
2315
|
+
static int sb_xfer_hold(sb_r2_transfer_ctx* ctx) {
|
|
2316
|
+
for (int i = 0; i < SB_XFER_MAX; i++) if (!sb_xfer[i]) { sb_xfer[i] = ctx; return i + 1; }
|
|
2317
|
+
return 0;
|
|
2318
|
+
}
|
|
2319
|
+
static sb_r2_transfer_ctx* sb_xfer_get(int handle) {
|
|
2320
|
+
return handle >= 1 && handle <= SB_XFER_MAX ? sb_xfer[handle - 1] : 0;
|
|
2321
|
+
}
|
|
2322
|
+
extern long long sb_reqbody_poll(unsigned int);
|
|
2323
|
+
extern size_t sb_reqbody_take(unsigned int, char*, size_t);
|
|
2324
|
+
// Claims an upload ticket: the HTTP status (0: claimed, HANDLE set).
|
|
2325
|
+
static int sb_xfer_upload_open(const char* bucket, const char* token, size_t declared, int* handle) {
|
|
2326
|
+
sb_r2_transfer_ctx* ctx = 0;
|
|
2327
|
+
*handle = 0;
|
|
2328
|
+
int status = sb_r2_transfer_open(bucket, token, declared, &ctx);
|
|
2329
|
+
if (status != 0 || !ctx) return status ? status : 500;
|
|
2330
|
+
*handle = sb_xfer_hold(ctx);
|
|
2331
|
+
if (!*handle) { sb_r2_transfer_abort(ctx); return 503; }
|
|
2332
|
+
return 0;
|
|
2333
|
+
}
|
|
2334
|
+
// One step of an upload from request body BODY: 'd' some bytes moved (*arg), 'w' none yet (wait for the
|
|
2335
|
+
// body's timer), 'e' the body ended (finish), 'f' the body failed (the client went away, abort), 's' a
|
|
2336
|
+
// status refused them (*arg; the ticket is released). A step stops after 1 MiB so other requests run.
|
|
2337
|
+
static char sb_xfer_upload_step(int handle, unsigned int body, long* arg) {
|
|
2338
|
+
sb_r2_transfer_ctx* ctx = sb_xfer_get(handle);
|
|
2339
|
+
*arg = 0;
|
|
2340
|
+
if (!ctx) { *arg = 500; return 's'; }
|
|
2341
|
+
char piece[65536];
|
|
2342
|
+
long moved = 0;
|
|
2343
|
+
while (moved < (1 << 20)) {
|
|
2344
|
+
long long ready = sb_reqbody_poll(body);
|
|
2345
|
+
if (ready == -1) { *arg = moved; return moved ? 'd' : 'e'; }
|
|
2346
|
+
if (ready == -2) { *arg = moved; return moved ? 'd' : 'f'; }
|
|
2347
|
+
if (ready == 0) { *arg = moved; return moved ? 'd' : 'w'; }
|
|
2348
|
+
size_t n = sb_reqbody_take(body, piece, sizeof(piece));
|
|
2349
|
+
if (!n) { *arg = moved; return moved ? 'd' : 'w'; }
|
|
2350
|
+
int status = sb_r2_transfer_write(ctx, piece, n);
|
|
2351
|
+
if (status != 0) { sb_r2_transfer_abort(ctx); sb_xfer[handle - 1] = 0; *arg = status; return 's'; }
|
|
2352
|
+
moved += (long)n;
|
|
2353
|
+
}
|
|
2354
|
+
*arg = moved;
|
|
2355
|
+
return 'd';
|
|
2356
|
+
}
|
|
2357
|
+
// A body that arrived whole (a request of a host that buffers it): written in one go.
|
|
2358
|
+
static int sb_xfer_upload_write(int handle, const char* bytes, size_t length) {
|
|
2359
|
+
sb_r2_transfer_ctx* ctx = sb_xfer_get(handle);
|
|
2360
|
+
if (!ctx) return 500;
|
|
2361
|
+
int status = sb_r2_transfer_write(ctx, bytes, length);
|
|
2362
|
+
if (status != 0) { sb_r2_transfer_abort(ctx); sb_xfer[handle - 1] = 0; }
|
|
2363
|
+
return status;
|
|
2364
|
+
}
|
|
2365
|
+
static int sb_xfer_finish(int handle) {
|
|
2366
|
+
sb_r2_transfer_ctx* ctx = sb_xfer_get(handle);
|
|
2367
|
+
if (!ctx) return 500;
|
|
2368
|
+
sb_xfer[handle - 1] = 0;
|
|
2369
|
+
return sb_r2_transfer_finish(ctx);
|
|
2370
|
+
}
|
|
2371
|
+
static void sb_xfer_close(int handle) {
|
|
2372
|
+
sb_r2_transfer_ctx* ctx = sb_xfer_get(handle);
|
|
2373
|
+
if (!ctx) return;
|
|
2374
|
+
sb_xfer[handle - 1] = 0;
|
|
2375
|
+
sb_r2_transfer_abort(ctx);
|
|
2376
|
+
}
|
|
2377
|
+
// Claims a download ticket: the HTTP status (0: claimed, HANDLE, *size and *etag set).
|
|
2378
|
+
static int sb_xfer_download_open(const char* bucket, const char* token, int* handle, size_t* size, char* etag, size_t etag_cap) {
|
|
2379
|
+
sb_r2_transfer_ctx* ctx = 0;
|
|
2380
|
+
*handle = 0;
|
|
2381
|
+
int status = sb_r2_transfer_download_open(bucket, token, &ctx);
|
|
2382
|
+
if (status != 0 || !ctx) return status ? status : 500;
|
|
2383
|
+
*handle = sb_xfer_hold(ctx);
|
|
2384
|
+
if (!*handle) { sb_r2_transfer_download_close(ctx); return 503; }
|
|
2385
|
+
*size = sb_r2_transfer_download_size(ctx);
|
|
2386
|
+
snprintf(etag, etag_cap, "%s", sb_r2_transfer_download_etag(ctx));
|
|
2387
|
+
return 0;
|
|
2388
|
+
}
|
|
2389
|
+
static void sb_xfer_download_close(int handle) {
|
|
2390
|
+
sb_r2_transfer_ctx* ctx = sb_xfer_get(handle);
|
|
2391
|
+
if (!ctx) return;
|
|
2392
|
+
sb_xfer[handle - 1] = 0;
|
|
2393
|
+
sb_r2_transfer_download_close(ctx);
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2122
2396
|
// LTX capture (#141) failed after SQLite committed: the write is in the database but not in
|
|
2123
2397
|
// the replication log, and the database refuses writes until the process restarts. Said as
|
|
2124
2398
|
// such (SQLITE_IOERR), never as SQLite's own message, which still reads "not an error".
|
|
@@ -2127,7 +2401,7 @@ void sb_r2_transfer_download_close(sb_r2_transfer_ctx* ctx) { if (ctx) { if (ctx
|
|
|
2127
2401
|
// every write is refused (a same-size UPDATE or a DELETE still adds an LTX file to the
|
|
2128
2402
|
// spool, which counts toward the quota). Reads and transaction control still run.
|
|
2129
2403
|
#define SB_QUOTA_FULL "storage quota exceeded: writes are refused while this app's data, including its replication log, is over its quota"
|
|
2130
|
-
// sb_quota_frozen (per database slot) is declared with
|
|
2404
|
+
// sb_quota_frozen (per database slot) is declared with sb_db_close, which clears it.
|
|
2131
2405
|
static int sb_db_capture_failed(const char* path) {
|
|
2132
2406
|
for (int i = 0; i < sb_db_count; i++) if (strcmp(sb_db_names[i], path) == 0) return sb_ltx_states[i].failed;
|
|
2133
2407
|
return 0;
|
|
@@ -2444,6 +2718,7 @@ static char* sb_sql_run(const char* path, const char* sql, const char* params) {
|
|
|
2444
2718
|
|
|
2445
2719
|
// The storage quota (docs: an app's data directory is capped): bytes in regular files under
|
|
2446
2720
|
// dir, symlinks not followed. Bounded by depth; an unreadable entry counts as 0.
|
|
2721
|
+
static long sb_dir_files; // regular files counted by the last walk (the quota scan's interval grows with it)
|
|
2447
2722
|
static long long sb_dir_bytes(const char* dir, int depth) {
|
|
2448
2723
|
DIR* d = opendir(dir);
|
|
2449
2724
|
if (!d || depth > 16) { if (d) closedir(d); return 0; }
|
|
@@ -2455,7 +2730,7 @@ static long long sb_dir_bytes(const char* dir, int depth) {
|
|
|
2455
2730
|
if (snprintf(path, sizeof(path), "%s/%s", dir, e->d_name) >= (int)sizeof(path)) continue;
|
|
2456
2731
|
struct stat st;
|
|
2457
2732
|
if (lstat(path, &st)) continue;
|
|
2458
|
-
if (S_ISREG(st.st_mode)) total += (long long)st.st_size;
|
|
2733
|
+
if (S_ISREG(st.st_mode)) { total += (long long)st.st_size; sb_dir_files++; }
|
|
2459
2734
|
else if (S_ISDIR(st.st_mode)) total += sb_dir_bytes(path, depth + 1);
|
|
2460
2735
|
}
|
|
2461
2736
|
closedir(d);
|
|
@@ -4342,6 +4617,112 @@ function __sbR2GetRaw(path, bucket, key, blobId) {
|
|
|
4342
4617
|
return res;
|
|
4343
4618
|
}
|
|
4344
4619
|
|
|
4620
|
+
// R2 get without the whole object: fills VIEW (a fresh Uint8Array) with the bytes at OFFSET of the object's
|
|
4621
|
+
// file. Returns how many it read (fewer at the end of the object), or -1 when the file cannot be read.
|
|
4622
|
+
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
4623
|
+
function __sbR2ReadInto(path, bucket, key, blobId, offset, view) {
|
|
4624
|
+
let got = -1;
|
|
4625
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4626
|
+
Porffor.c`
|
|
4627
|
+
const char* __p; size_t __pl; char* __po = 0;
|
|
4628
|
+
porf_native_fetch_read_value(path, &__p, &__pl, &__po);
|
|
4629
|
+
char* __path = (char*)malloc(__pl + 1); memcpy(__path, __p, __pl); __path[__pl] = 0;
|
|
4630
|
+
if (__po) free(__po);
|
|
4631
|
+
|
|
4632
|
+
const char* __bk; size_t __bkl; char* __bko = 0;
|
|
4633
|
+
porf_native_fetch_read_value(bucket, &__bk, &__bkl, &__bko);
|
|
4634
|
+
char* __bucket = (char*)malloc(__bkl + 1); memcpy(__bucket, __bk, __bkl); __bucket[__bkl] = 0;
|
|
4635
|
+
if (__bko) free(__bko);
|
|
4636
|
+
|
|
4637
|
+
const char* __k; size_t __kl; char* __ko = 0;
|
|
4638
|
+
porf_native_fetch_read_value(key, &__k, &__kl, &__ko);
|
|
4639
|
+
char* __key = (char*)malloc(__kl + 1); memcpy(__key, __k, __kl); __key[__kl] = 0;
|
|
4640
|
+
if (__ko) free(__ko);
|
|
4641
|
+
|
|
4642
|
+
const char* __g; size_t __gl; char* __go = 0;
|
|
4643
|
+
porf_native_fetch_read_value(blobId, &__g, &__gl, &__go);
|
|
4644
|
+
char* __generation = (char*)malloc(__gl + 1); memcpy(__generation, __g, __gl); __generation[__gl] = 0;
|
|
4645
|
+
if (__go) free(__go);
|
|
4646
|
+
|
|
4647
|
+
long __n = -1;
|
|
4648
|
+
if (view.type == 81 && offset.type == 1 && offset.val >= 0) {
|
|
4649
|
+
const u32 __ta = (u32)view.val;
|
|
4650
|
+
const int32_t __len = *(int32_t*)(MEM + __ta);
|
|
4651
|
+
const u32 __at = *(u32*)(MEM + __ta + 4) + 4;
|
|
4652
|
+
if (__len >= 0) __n = sb_r2_read_into_c(__path, __bucket, __key, __generation, (size_t)offset.val, (char*)(MEM + __at), (size_t)__len);
|
|
4653
|
+
}
|
|
4654
|
+
free(__path); free(__bucket); free(__key); free(__generation);
|
|
4655
|
+
got = (f64)__n;
|
|
4656
|
+
`;
|
|
4657
|
+
return got;
|
|
4658
|
+
}
|
|
4659
|
+
|
|
4660
|
+
// Transfer tickets served by the worker (see sb_xfer_* above). op: "u" claim an upload ticket (a = bucket, b = token,
|
|
4661
|
+
// n = the declared length; "0 HANDLE" or "STATUS"), "s" a step of the upload from body id n ("d BYTES", "w", "e", "f"
|
|
4662
|
+
// or "s STATUS"), "b" write the bytestring in a (a buffered body; the status), "f" finish ("STATUS"), "x" release the
|
|
4663
|
+
// handle in n, "d" claim a download ticket ("0 HANDLE SIZE ETAG" or "STATUS"), "r" read the object's file at offset n
|
|
4664
|
+
// into the Uint8Array in view (handle in a; the count), "c" release a download handle. Returns a short string.
|
|
4665
|
+
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
4666
|
+
function __sbR2XferOp(op, a, b, n, view) {
|
|
4667
|
+
let out = "";
|
|
4668
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4669
|
+
Porffor.c`
|
|
4670
|
+
const char* __o; size_t __ol; char* __oo = 0;
|
|
4671
|
+
porf_native_fetch_read_value(op, &__o, &__ol, &__oo);
|
|
4672
|
+
char __op = __ol ? __o[0] : 0;
|
|
4673
|
+
if (__oo) free(__oo);
|
|
4674
|
+
|
|
4675
|
+
const char* __a; size_t __al; char* __ao = 0;
|
|
4676
|
+
porf_native_fetch_read_value(a, &__a, &__al, &__ao);
|
|
4677
|
+
char* __as = (char*)malloc(__al + 1); memcpy(__as, __a, __al); __as[__al] = 0;
|
|
4678
|
+
if (__ao) free(__ao);
|
|
4679
|
+
const char* __b; size_t __bl; char* __bo = 0;
|
|
4680
|
+
porf_native_fetch_read_value(b, &__b, &__bl, &__bo);
|
|
4681
|
+
char* __bs = (char*)malloc(__bl + 1); memcpy(__bs, __b, __bl); __bs[__bl] = 0;
|
|
4682
|
+
if (__bo) free(__bo);
|
|
4683
|
+
|
|
4684
|
+
const double __n = n.type == 1 ? n.val : 0;
|
|
4685
|
+
const int __handle = (int)strtol(__as, 0, 10);
|
|
4686
|
+
char __r[160]; __r[0] = 0;
|
|
4687
|
+
if (__op == 'u') {
|
|
4688
|
+
int __h = 0;
|
|
4689
|
+
int __s = sb_xfer_upload_open(__as, __bs, (size_t)__n, &__h);
|
|
4690
|
+
if (__s == 0) snprintf(__r, sizeof(__r), "0 %d", __h); else snprintf(__r, sizeof(__r), "%d", __s);
|
|
4691
|
+
} else if (__op == 's') {
|
|
4692
|
+
long __moved = 0;
|
|
4693
|
+
char __k = sb_xfer_upload_step(__handle, (unsigned int)__n, &__moved);
|
|
4694
|
+
snprintf(__r, sizeof(__r), "%c %ld", __k, __moved);
|
|
4695
|
+
} else if (__op == 'b') {
|
|
4696
|
+
const char* __d = 0; size_t __dl = 0;
|
|
4697
|
+
int __s = 500;
|
|
4698
|
+
if (porf_native_fetch_read_raw_bytes(b, &__d, &__dl) == 0) __s = sb_xfer_upload_write(__handle, __d, __dl);
|
|
4699
|
+
snprintf(__r, sizeof(__r), "%d", __s);
|
|
4700
|
+
} else if (__op == 'f') snprintf(__r, sizeof(__r), "%d", sb_xfer_finish(__handle));
|
|
4701
|
+
else if (__op == 'x') sb_xfer_close((int)__n);
|
|
4702
|
+
else if (__op == 'd') {
|
|
4703
|
+
int __h = 0; size_t __size = 0; char __etag[80]; __etag[0] = 0;
|
|
4704
|
+
int __s = sb_xfer_download_open(__as, __bs, &__h, &__size, __etag, sizeof(__etag));
|
|
4705
|
+
if (__s == 0) snprintf(__r, sizeof(__r), "0 %d %zu %s", __h, __size, __etag); else snprintf(__r, sizeof(__r), "%d", __s);
|
|
4706
|
+
} else if (__op == 'r') {
|
|
4707
|
+
long __got = -1;
|
|
4708
|
+
sb_r2_transfer_ctx* __ctx = sb_xfer_get(__handle);
|
|
4709
|
+
if (__ctx && view.type == 81) {
|
|
4710
|
+
const u32 __ta = (u32)view.val;
|
|
4711
|
+
const int32_t __len = *(int32_t*)(MEM + __ta);
|
|
4712
|
+
const u32 __at = *(u32*)(MEM + __ta + 4) + 4;
|
|
4713
|
+
if (__len >= 0) __got = (long)sb_r2_transfer_download_read(__ctx, (size_t)__n, (char*)(MEM + __at), (size_t)__len);
|
|
4714
|
+
}
|
|
4715
|
+
snprintf(__r, sizeof(__r), "%ld", __got);
|
|
4716
|
+
} else if (__op == 'c') sb_xfer_download_close((int)__n);
|
|
4717
|
+
free(__as); free(__bs);
|
|
4718
|
+
out = porf_box((f64)porf_native_fetch_alloc_bytestring(__r, strlen(__r)), 195);
|
|
4719
|
+
`;
|
|
4720
|
+
return out;
|
|
4721
|
+
}
|
|
4722
|
+
// This transport's workers serve transfer tickets themselves, on a host that forwards the path (sproutd); a
|
|
4723
|
+
// standalone executable's native server answers them before JavaScript runs, and never gets here.
|
|
4724
|
+
__sbR2TransferOn = true;
|
|
4725
|
+
|
|
4345
4726
|
// R2 delete: also removes the object's own file, not just the metadata row
|
|
4346
4727
|
// (#56 — sb_r2_delete_blob is best-effort, a missing file is not an error).
|
|
4347
4728
|
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
@@ -4399,7 +4780,7 @@ function __sbSqlScriptRaw(path, sql) {
|
|
|
4399
4780
|
return res;
|
|
4400
4781
|
}
|
|
4401
4782
|
|
|
4402
|
-
// The bytes under a directory, as
|
|
4783
|
+
// The bytes under a directory and the number of regular files, as "BYTES FILES" (sb_dir_bytes).
|
|
4403
4784
|
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
4404
4785
|
function __sbDirBytesRaw(dir) {
|
|
4405
4786
|
let res = "";
|
|
@@ -4409,13 +4790,50 @@ function __sbDirBytesRaw(dir) {
|
|
|
4409
4790
|
porf_native_fetch_read_value(dir, &__d, &__dl, &__do);
|
|
4410
4791
|
char* __dir = (char*)malloc(__dl + 1); memcpy(__dir, __d, __dl); __dir[__dl] = 0;
|
|
4411
4792
|
if (__do) free(__do);
|
|
4412
|
-
char __n[
|
|
4793
|
+
char __n[64]; sb_dir_files = 0; long long __b = sb_dir_bytes(__dir, 0); int __k = snprintf(__n, sizeof(__n), "%lld %ld", __b, sb_dir_files);
|
|
4413
4794
|
free(__dir);
|
|
4414
4795
|
res = porf_box((f64)porf_native_fetch_alloc_bytestring(__n, (size_t)__k), 195);
|
|
4415
4796
|
`;
|
|
4416
4797
|
return res;
|
|
4417
4798
|
}
|
|
4418
4799
|
|
|
4800
|
+
// Closes the handle of one database path (sb_db_close_path): 1 closed, 0 not open, -1 left open.
|
|
4801
|
+
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
4802
|
+
function __sbDbCloseRaw(path) {
|
|
4803
|
+
let status = 0;
|
|
4804
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4805
|
+
Porffor.c`
|
|
4806
|
+
const char* __p; size_t __pl; char* __po = 0;
|
|
4807
|
+
porf_native_fetch_read_value(path, &__p, &__pl, &__po);
|
|
4808
|
+
char* __path = (char*)malloc(__pl + 1); memcpy(__path, __p, __pl); __path[__pl] = 0;
|
|
4809
|
+
if (__po) free(__po);
|
|
4810
|
+
status = (f64)sb_db_close_path(__path);
|
|
4811
|
+
free(__path);
|
|
4812
|
+
`;
|
|
4813
|
+
return status;
|
|
4814
|
+
}
|
|
4815
|
+
|
|
4816
|
+
// The handle table's counters, one JSON line (sb_db_stats).
|
|
4817
|
+
function __sbDbStatsRaw() {
|
|
4818
|
+
let res = "";
|
|
4819
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4820
|
+
Porffor.c`
|
|
4821
|
+
char __n[512]; int __k = sb_db_stats(__n, sizeof(__n));
|
|
4822
|
+
res = porf_box((f64)porf_native_fetch_alloc_bytestring(__n, (size_t)__k), 195);
|
|
4823
|
+
`;
|
|
4824
|
+
return res;
|
|
4825
|
+
}
|
|
4826
|
+
|
|
4827
|
+
// Hands SQLite's cached pages back (sb_db_release_all); returns the bytes released.
|
|
4828
|
+
function __sbDbReleaseRaw() {
|
|
4829
|
+
let freed = 0;
|
|
4830
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4831
|
+
Porffor.c`
|
|
4832
|
+
freed = (f64)sb_db_release_all();
|
|
4833
|
+
`;
|
|
4834
|
+
return freed;
|
|
4835
|
+
}
|
|
4836
|
+
|
|
4419
4837
|
// "captured" of __sbD1DbRaw as a number: no text made in C and parsed here, on every D1 query.
|
|
4420
4838
|
// oxlint-disable-next-line no-unused-vars -- read inside the RawC block below, not by JS.
|
|
4421
4839
|
function __sbD1Captured_(path) {
|
|
@@ -4761,6 +5179,9 @@ function __sbDir() {
|
|
|
4761
5179
|
var __sbQuotaMax = -1;
|
|
4762
5180
|
var __sbQuotaUsed = 0;
|
|
4763
5181
|
var __sbQuotaAt = -1;
|
|
5182
|
+
// Milliseconds between scans of the data directory: one lstat per file, so 1 s per 1,000 files and at most 10 s (7 ms of
|
|
5183
|
+
// walking at 5,000 files, 20 ms at 15,000, measured). Writes can overshoot the quota by up to that long.
|
|
5184
|
+
var __sbQuotaEvery = 1000;
|
|
4764
5185
|
function __sbQuotaLimit() {
|
|
4765
5186
|
if (__sbQuotaMax < 0) {
|
|
4766
5187
|
const v = Number(__sbEnv("SB_STORAGE_MAX_BYTES"));
|
|
@@ -4770,8 +5191,10 @@ function __sbQuotaLimit() {
|
|
|
4770
5191
|
}
|
|
4771
5192
|
function __sbQuotaFull() {
|
|
4772
5193
|
const now = Date.now();
|
|
4773
|
-
if (__sbQuotaAt < 0 || now - __sbQuotaAt >=
|
|
4774
|
-
|
|
5194
|
+
if (__sbQuotaAt < 0 || now - __sbQuotaAt >= __sbQuotaEvery) {
|
|
5195
|
+
const scan = String(__sbDirBytesRaw(__sbDir())).split(" ");
|
|
5196
|
+
__sbQuotaUsed = Number(scan[0]) || 0;
|
|
5197
|
+
__sbQuotaEvery = Math.min(10000, Math.max(1000, Math.ceil((Number(scan[1]) || 0) / 1000) * 1000));
|
|
4775
5198
|
__sbQuotaAt = now;
|
|
4776
5199
|
}
|
|
4777
5200
|
return __sbQuotaUsed >= __sbQuotaLimit();
|
|
@@ -5128,10 +5551,8 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
5128
5551
|
let path = String(msg.path || "/");
|
|
5129
5552
|
if (path.charAt(0) !== "/") path = "/" + path;
|
|
5130
5553
|
const redirects = (bundle.manifest && bundle.manifest.redirects) || [];
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
if (location !== null) return { ok: true, found: false, status: redirects[i].status, location };
|
|
5134
|
-
}
|
|
5554
|
+
const redirect = __sbFindAssetRedirect(redirects, path);
|
|
5555
|
+
if (redirect !== null) return { ok: true, found: false, status: redirect.status, location: redirect.location };
|
|
5135
5556
|
|
|
5136
5557
|
let key = null;
|
|
5137
5558
|
if (path.charAt(path.length - 1) === "/") {
|
|
@@ -5294,7 +5715,7 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
5294
5715
|
// run like a strict D1 query (BLOBs as hex, D1_ERROR text the shim rewords).
|
|
5295
5716
|
if (op === "do.sql") {
|
|
5296
5717
|
const path = __sbDOSqlPath(msg.cls, msg.file);
|
|
5297
|
-
// Its handle may have been closed and reopened since (
|
|
5718
|
+
// Its handle may have been closed and reopened since (the slot LRU, an eviction), which drops the
|
|
5298
5719
|
// size cap: set it again.
|
|
5299
5720
|
delete __sbD1Capped[path];
|
|
5300
5721
|
__sbD1Quota(path);
|
|
@@ -5303,6 +5724,16 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
5303
5724
|
return { ok: true, columns: r.cols, rows: r.rows, meta: { changes: r.changes } };
|
|
5304
5725
|
}
|
|
5305
5726
|
|
|
5727
|
+
// A Durable Object instance leaving memory closes its SQLite handle (plan 018): the last connection checkpoints, so the
|
|
5728
|
+
// evicted object is one clean file. A no-op when it is not open; closed: -1 means a transaction or a statement kept it open.
|
|
5729
|
+
if (op === "do.sql.close") {
|
|
5730
|
+
const path = __sbDOSqlPath(msg.cls, msg.file);
|
|
5731
|
+
return { ok: true, closed: __sbDbCloseRaw(path) };
|
|
5732
|
+
}
|
|
5733
|
+
|
|
5734
|
+
if (op === "do.stats") return { ok: true, stats: JSON.parse(__sbDbStatsRaw()) };
|
|
5735
|
+
if (op === "do.release") return { ok: true, freed: __sbDbReleaseRaw() };
|
|
5736
|
+
|
|
5306
5737
|
if (op === "d1.backup") {
|
|
5307
5738
|
const src = __sbD1Path(String(msg.db));
|
|
5308
5739
|
// A caller-supplied name must be a plain basename; anything else falls back
|
|
@@ -5955,6 +6386,19 @@ globalThis.__sbR2Get = function (bucket, key) {
|
|
|
5955
6386
|
};
|
|
5956
6387
|
};
|
|
5957
6388
|
|
|
6389
|
+
// The object's metadata and where its bytes are, without the bytes: get() reads them as the caller asks for them
|
|
6390
|
+
// (range windows and a body stream), through __sbR2ReadRange.
|
|
6391
|
+
globalThis.__sbR2Open = function (bucket, key) {
|
|
6392
|
+
const meta = __sbEmbeddedDispatch({ op: "r2.head", bucket, key });
|
|
6393
|
+
if (!meta.found) return { found: false };
|
|
6394
|
+
return { found: true, object: meta.object, blobId: String(meta.blobId || "") };
|
|
6395
|
+
};
|
|
6396
|
+
|
|
6397
|
+
// Fills the Uint8Array VIEW with the object's bytes from OFFSET; the count read, or -1.
|
|
6398
|
+
globalThis.__sbR2ReadRange = function (bucket, key, blobId, offset, view) {
|
|
6399
|
+
return __sbR2ReadInto(__sbStore(), String(bucket), String(key), String(blobId), offset, view);
|
|
6400
|
+
};
|
|
6401
|
+
|
|
5958
6402
|
globalThis.__sbR2MultipartPut = function (bucket, key, uploadId, partNumber, body) {
|
|
5959
6403
|
__sbQuotaCheck("R2 uploadPart()");
|
|
5960
6404
|
__sbEnsureSchema();
|
|
@@ -6173,7 +6617,7 @@ async function __sbQueuePass(handlers, queues) {
|
|
|
6173
6617
|
// From 2026-11-05 a consumer that runs past the 30 s lease keeps its messages: the lease
|
|
6174
6618
|
// is renewed while it runs, so they are not delivered a second time or lose their ack.
|
|
6175
6619
|
const beat = __sbQueuesFixed()
|
|
6176
|
-
?
|
|
6620
|
+
? __sbSetInterval(() => {
|
|
6177
6621
|
__sbSql(store, "UPDATE mq SET visible_at=? WHERE queue=? AND claim_token=? AND visible_at>?", [Date.now() + 30000, name, claim, Date.now()]);
|
|
6178
6622
|
}, 10000)
|
|
6179
6623
|
: null;
|
|
@@ -6183,7 +6627,7 @@ async function __sbQueuePass(handlers, queues) {
|
|
|
6183
6627
|
if (result && __sbIsFn(result.then)) result = await result;
|
|
6184
6628
|
if (__sbTasks.length) await __sbDrainWaitUntil(__sbTasks);
|
|
6185
6629
|
} finally {
|
|
6186
|
-
if (beat !== null)
|
|
6630
|
+
if (beat !== null) __sbClearInterval(beat);
|
|
6187
6631
|
}
|
|
6188
6632
|
__sbQueueSettle(store, name, claim, result);
|
|
6189
6633
|
}
|
|
@@ -6296,6 +6740,9 @@ async function __sbAlarmPass() {
|
|
|
6296
6740
|
__sbSql(store, "ROLLBACK", []);
|
|
6297
6741
|
throw error;
|
|
6298
6742
|
}
|
|
6743
|
+
// From the eviction date the claimed batch runs at once (each alarm through its own object's gate; one alarm() per object at a
|
|
6744
|
+
// time is the gate and the single row), the claim is renewed while it runs, and the row's attempts reach alarm().
|
|
6745
|
+
if (__sbDOEvictOn()) return __sbAlarmRunBatch(store, claim, due.rows);
|
|
6299
6746
|
for (let i = 0; i < due.rows.length; i++) {
|
|
6300
6747
|
const cls = due.rows[i][0];
|
|
6301
6748
|
const id = due.rows[i][1];
|
|
@@ -6334,6 +6781,46 @@ async function __sbAlarmPass() {
|
|
|
6334
6781
|
}
|
|
6335
6782
|
}
|
|
6336
6783
|
|
|
6784
|
+
async function __sbAlarmRunBatch(store, claim, rows) {
|
|
6785
|
+
const beat = __sbSetInterval(() => {
|
|
6786
|
+
__sbSql(store, "UPDATE do_alarm_delivery SET expires=? WHERE token=? AND expires>?", [Date.now() + 30000, claim, Date.now()]);
|
|
6787
|
+
}, 10000);
|
|
6788
|
+
try {
|
|
6789
|
+
const runs = [];
|
|
6790
|
+
for (let i = 0; i < rows.length; i++) runs.push(__sbAlarmDeliver(store, claim, rows[i]));
|
|
6791
|
+
for (let i = 0; i < runs.length; i++) await runs[i];
|
|
6792
|
+
} finally {
|
|
6793
|
+
__sbClearInterval(beat);
|
|
6794
|
+
}
|
|
6795
|
+
}
|
|
6796
|
+
|
|
6797
|
+
// One claimed alarm: construct the object if it is evicted (its constructor runs before alarm()), run alarm() through its gate, and
|
|
6798
|
+
// settle the delivery row. A call that could not get a slot within the wait is not a failed attempt: the row is released uncounted.
|
|
6799
|
+
async function __sbAlarmDeliver(store, claim, row) {
|
|
6800
|
+
const cls = row[0];
|
|
6801
|
+
const id = row[1];
|
|
6802
|
+
try {
|
|
6803
|
+
await __sbDORun(cls, id, undefined, (instance) => (__sbIsFn(instance.alarm) ? __sbDOAlarmCall(instance, cls, id, row[2], row[3]) : undefined), true);
|
|
6804
|
+
__sbSql(store, "DELETE FROM do_alarm_delivery WHERE cls=? AND id=? AND token=? AND expires>?", [cls, id, claim, Date.now()]);
|
|
6805
|
+
} catch (error) {
|
|
6806
|
+
if (error && error.overloaded === true) {
|
|
6807
|
+
__sbSql(store, "UPDATE do_alarm_delivery SET expires=?, attempts=attempts-1 WHERE cls=? AND id=? AND token=? AND expires>?", [
|
|
6808
|
+
Date.now() + 1000,
|
|
6809
|
+
cls,
|
|
6810
|
+
id,
|
|
6811
|
+
claim,
|
|
6812
|
+
Date.now(),
|
|
6813
|
+
]);
|
|
6814
|
+
return;
|
|
6815
|
+
}
|
|
6816
|
+
const scheduled = __sbSql(store, "SELECT 1 FROM do_alarm WHERE cls=? AND id=?", [cls, id]);
|
|
6817
|
+
if (scheduled.rows.length)
|
|
6818
|
+
__sbSql(store, "DELETE FROM do_alarm_delivery WHERE cls=? AND id=? AND token=? AND expires>?", [cls, id, claim, Date.now()]);
|
|
6819
|
+
else
|
|
6820
|
+
__sbSql(store, "UPDATE do_alarm_delivery SET expires=? WHERE cls=? AND id=? AND token=? AND expires>?", [Date.now() + 5000, cls, id, claim, Date.now()]);
|
|
6821
|
+
}
|
|
6822
|
+
}
|
|
6823
|
+
|
|
6337
6824
|
// B22: on a host that delivers triggers (sproutd), what this worker consumes. Set by
|
|
6338
6825
|
// __sbStartLocalTriggers; null when the host does not drive this worker's triggers.
|
|
6339
6826
|
var __sbLocal = null;
|
|
@@ -6420,7 +6907,7 @@ globalThis.__sbStartLocalTriggers = function (handlers, bindings) {
|
|
|
6420
6907
|
}
|
|
6421
6908
|
|
|
6422
6909
|
if (scheduled) {
|
|
6423
|
-
|
|
6910
|
+
__sbSetInterval(async function () {
|
|
6424
6911
|
const now = new Date();
|
|
6425
6912
|
const stamp =
|
|
6426
6913
|
now.getUTCFullYear() +
|
|
@@ -6460,7 +6947,7 @@ globalThis.__sbStartLocalTriggers = function (handlers, bindings) {
|
|
|
6460
6947
|
}, 15000);
|
|
6461
6948
|
}
|
|
6462
6949
|
|
|
6463
|
-
if (queues.length > 0)
|
|
6950
|
+
if (queues.length > 0) __sbSetInterval(() => __sbQueuePass(handlers, queues), 500);
|
|
6464
6951
|
|
|
6465
|
-
if (dos.length > 0)
|
|
6952
|
+
if (dos.length > 0) __sbSetInterval(() => __sbAlarmPass(), 500);
|
|
6466
6953
|
};
|