@sproutboat/runtime 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/LTX_REPLICATION.md +124 -0
- package/README.md +7 -0
- package/package.json +2 -2
- package/src/native-fetch-prelude.js +84 -14
- package/src/transport-embedded.js +1077 -4
- package/test/native-fetch-request.ts +305 -0
- package/test/native-ltx.ts +359 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { ensurePorffor } from "../../toolchain/src/acquire";
|
|
6
|
+
import { createBroker, listen } from "../../wire/src/broker";
|
|
7
|
+
import { EMPTY_BINDINGS, preludePath, TRANSPORT_MARKER, transportPath, wrapNativeFetchHandler } from "../src/wrap";
|
|
8
|
+
|
|
9
|
+
// fetch(Request), fetch(Request, init) and new Request(request, init) on a compiled sprout,
|
|
10
|
+
// through the broker transport, against what workerd does (the platform's
|
|
11
|
+
// tests/service-bindings suite holds workerd's answers). The same handler is compiled at
|
|
12
|
+
// compatibility date 2026-10-23, where Request follows Fetch as workerd does, and at
|
|
13
|
+
// 2026-10-22, where an app keeps the older behaviour: a copied Request shares its body,
|
|
14
|
+
// headers stay mutable and any method name passes.
|
|
15
|
+
const handler = `
|
|
16
|
+
const textStream = (...parts) =>
|
|
17
|
+
new ReadableStream({
|
|
18
|
+
start(c) {
|
|
19
|
+
for (const p of parts) c.enqueue(new TextEncoder().encode(p));
|
|
20
|
+
c.close();
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
const attempt = async (step) => {
|
|
24
|
+
try {
|
|
25
|
+
return await step();
|
|
26
|
+
} catch (e) {
|
|
27
|
+
return e.name + ": " + e.message;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export default {
|
|
32
|
+
async fetch(request, env) {
|
|
33
|
+
const url = new URL(request.url);
|
|
34
|
+
const up = url.searchParams.get("up");
|
|
35
|
+
const echo = async (res) => res.json();
|
|
36
|
+
if (url.pathname === "/ping") return new Response("pong");
|
|
37
|
+
if (url.pathname === "/fetch") {
|
|
38
|
+
const viaRequest = await echo(await fetch(new Request(up + "/echo", { method: "PUT", headers: { "x-from": "request" }, body: "req body" })));
|
|
39
|
+
const plusInit = await echo(
|
|
40
|
+
await fetch(new Request(up + "/echo", { method: "PUT", headers: { "x-from": "request" }, body: "req body" }), { method: "PATCH", headers: { "x-from": "init" } }),
|
|
41
|
+
);
|
|
42
|
+
const initBody = await echo(await fetch(new Request(up + "/echo", { method: "PUT", body: "req body" }), { body: "init body" }));
|
|
43
|
+
const stream = await echo(await fetch(new Request(up + "/echo", { method: "POST", body: textStream("in ", "a request"), duplex: "half" })));
|
|
44
|
+
const initStream = await echo(await fetch(up + "/echo", { method: "POST", body: textStream("in ", "init"), duplex: "half" }));
|
|
45
|
+
const initRequest = new Request("https://other/", { method: "POST", headers: { "x-from": "init-request" }, body: "init req" });
|
|
46
|
+
const asInit = { ...(await echo(await fetch(up + "/echo", initRequest))), initUsed: initRequest.bodyUsed };
|
|
47
|
+
const manual = (await fetch(new Request(up + "/redirect", { redirect: "manual" }))).status;
|
|
48
|
+
const aborted = await attempt(() => fetch(new Request(up + "/echo", { signal: AbortSignal.abort() })).then((r) => r.status));
|
|
49
|
+
const once = new Request(up + "/echo", { method: "POST", body: "once" });
|
|
50
|
+
const first = (await echo(await fetch(once))).body;
|
|
51
|
+
const usedUp = once.bodyUsed;
|
|
52
|
+
const again = await attempt(() => fetch(once).then(echo).then((e) => e.body));
|
|
53
|
+
const getBody = await attempt(() => fetch(up + "/echo", { body: "x" }).then((r) => r.status));
|
|
54
|
+
return Response.json({ viaRequest, plusInit, initBody, stream, initStream, asInit, manual, aborted, first, usedUp, again, getBody });
|
|
55
|
+
}
|
|
56
|
+
if (url.pathname === "/forward") {
|
|
57
|
+
// The incoming request's raw bytes reach the upstream intact.
|
|
58
|
+
const copied = await echo(await fetch(new Request(up + "/echo", request)));
|
|
59
|
+
return Response.json({ copied, incomingUsed: request.bodyUsed });
|
|
60
|
+
}
|
|
61
|
+
if (url.pathname === "/request") {
|
|
62
|
+
const a = new Request("https://peer/a", { method: "POST", headers: { "x-a": "1" }, body: "abc" });
|
|
63
|
+
const b = new Request(a, { headers: { "x-b": "2" } });
|
|
64
|
+
const copy = { aUsed: a.bodyUsed, aRead: await attempt(() => a.text()), b: await b.text(), headers: [...b.headers] };
|
|
65
|
+
const s = new Request("https://peer/s", { method: "POST", body: textStream("tee ", "me"), duplex: "half" });
|
|
66
|
+
const t = s.clone();
|
|
67
|
+
const tee = [await t.text(), await s.text()];
|
|
68
|
+
const controller = new AbortController();
|
|
69
|
+
const signalled = new Request(new Request("https://peer/", { signal: controller.signal }));
|
|
70
|
+
controller.abort();
|
|
71
|
+
const res = await fetch(url.searchParams.get("up") + "/echo");
|
|
72
|
+
await res.text();
|
|
73
|
+
return Response.json({
|
|
74
|
+
copy,
|
|
75
|
+
tee,
|
|
76
|
+
signal: [signalled.signal.aborted, new Request("https://peer/").signal.aborted],
|
|
77
|
+
getBody: await attempt(() => new Request("https://peer/", { body: "" }).method),
|
|
78
|
+
method: await attempt(() => new Request("https://peer/", { method: "custom" }).method),
|
|
79
|
+
redirect: await attempt(() => new Request("https://peer/", { redirect: "error" }).redirect),
|
|
80
|
+
incoming: await attempt(() => (request.headers.set("x-t", "1"), "mutable")),
|
|
81
|
+
fetched: await attempt(() => (res.headers.set("x-t", "1"), "mutable")),
|
|
82
|
+
copiedResponse: await attempt(() => (new Response(null, res).headers.set("x-t", "1"), "mutable")),
|
|
83
|
+
incomingBody: request.body === null ? "none" : typeof request.body,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return new Response("no route", { status: 404 });
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
`;
|
|
90
|
+
|
|
91
|
+
function freePort(): number {
|
|
92
|
+
const listener = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
|
|
93
|
+
const port = listener.port;
|
|
94
|
+
listener.stop(true);
|
|
95
|
+
return port;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const upstream = Bun.serve({
|
|
99
|
+
hostname: "127.0.0.1",
|
|
100
|
+
port: 0,
|
|
101
|
+
async fetch(request) {
|
|
102
|
+
const url = new URL(request.url);
|
|
103
|
+
if (url.pathname === "/redirect") return new Response(null, { status: 302, headers: { location: "/echo?hop=1" } });
|
|
104
|
+
const bytes = new Uint8Array(await request.arrayBuffer());
|
|
105
|
+
let binaryIntact = bytes.length === 256;
|
|
106
|
+
for (let i = 0; binaryIntact && i < 256; i++) binaryIntact = bytes[i] === i;
|
|
107
|
+
return Response.json({
|
|
108
|
+
method: request.method,
|
|
109
|
+
from: request.headers.get("x-from"),
|
|
110
|
+
type: request.headers.get("content-type"),
|
|
111
|
+
body: binaryIntact ? "bytes 0-255" : new TextDecoder().decode(bytes),
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
const up = `http://127.0.0.1:${upstream.port}`;
|
|
116
|
+
const workdir = await mkdtemp(join(tmpdir(), "sb-native-fetch-request-"));
|
|
117
|
+
const broker = createBroker({
|
|
118
|
+
db: ":memory:",
|
|
119
|
+
token: "native-fetch-request",
|
|
120
|
+
bindings: { ...EMPTY_BINDINGS, resources: {} },
|
|
121
|
+
egressAllow: ["127.0.0.1"],
|
|
122
|
+
});
|
|
123
|
+
const brokerServer = listen(broker, "127.0.0.1", 0);
|
|
124
|
+
const servers: ReturnType<typeof Bun.spawn>[] = [];
|
|
125
|
+
|
|
126
|
+
// Compiles the handler at `date` and starts it; resolves to a GET helper for it.
|
|
127
|
+
async function start(porffor: string, node: string, source: string, date: string) {
|
|
128
|
+
const port = freePort();
|
|
129
|
+
const generated = join(workdir, `fetch-request-${date}.generated.js`);
|
|
130
|
+
const binary = join(workdir, `fetch-request-${date}`);
|
|
131
|
+
await writeFile(generated, wrapNativeFetchHandler(handler, source, {}, EMPTY_BINDINGS, undefined, date));
|
|
132
|
+
const compile = Bun.spawn(
|
|
133
|
+
[node, resolve(porffor, "runtime/index.js"), "native", generated, "-o", binary, "-s", "-O0"],
|
|
134
|
+
{
|
|
135
|
+
cwd: workdir,
|
|
136
|
+
stdout: "pipe",
|
|
137
|
+
stderr: "pipe",
|
|
138
|
+
},
|
|
139
|
+
);
|
|
140
|
+
const [exitCode, stderr] = await Promise.all([compile.exited, new Response(compile.stderr).text()]);
|
|
141
|
+
assert.equal(exitCode, 0, `Porffor native compile at ${date} failed:\n${stderr}`);
|
|
142
|
+
const server = Bun.spawn([binary], {
|
|
143
|
+
cwd: workdir,
|
|
144
|
+
env: {
|
|
145
|
+
...process.env,
|
|
146
|
+
PORT: String(port),
|
|
147
|
+
SB_BROKER_PORT: String(brokerServer.port),
|
|
148
|
+
SB_BROKER_TOKEN: "native-fetch-request",
|
|
149
|
+
},
|
|
150
|
+
stdout: "ignore",
|
|
151
|
+
stderr: "inherit",
|
|
152
|
+
});
|
|
153
|
+
servers.push(server);
|
|
154
|
+
const call = async (path: string, init?: RequestInit) => {
|
|
155
|
+
const sep = path.includes("?") ? "&" : "?";
|
|
156
|
+
const res = await fetch(`http://127.0.0.1:${port}${path}${sep}up=${encodeURIComponent(up)}`, {
|
|
157
|
+
...init,
|
|
158
|
+
signal: AbortSignal.timeout(20_000),
|
|
159
|
+
});
|
|
160
|
+
const text = await res.text();
|
|
161
|
+
assert.equal(res.status, 200, `${date} ${path}: ${text}`);
|
|
162
|
+
return JSON.parse(text);
|
|
163
|
+
};
|
|
164
|
+
for (let i = 0; i < 50; i++) {
|
|
165
|
+
try {
|
|
166
|
+
const res = await fetch(`http://127.0.0.1:${port}/ping`);
|
|
167
|
+
if ((await res.text()) === "pong") return call;
|
|
168
|
+
} catch {
|
|
169
|
+
if (server.exitCode !== null) break;
|
|
170
|
+
await Bun.sleep(100);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
throw new Error(`compiled handler at ${date} did not start`);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
const porffor = await ensurePorffor({ cacheRoot: workdir });
|
|
178
|
+
const node = Bun.which("node");
|
|
179
|
+
if (!node) throw new Error("native fetch-request test needs node on PATH");
|
|
180
|
+
const [core, transport] = await Promise.all([
|
|
181
|
+
readFile(preludePath, "utf8"),
|
|
182
|
+
readFile(transportPath("broker"), "utf8"),
|
|
183
|
+
]);
|
|
184
|
+
assert(core.includes(TRANSPORT_MARKER), "runtime prelude is missing its transport marker");
|
|
185
|
+
const source = core.replace(TRANSPORT_MARKER, transport);
|
|
186
|
+
const [strict, legacy] = await Promise.all([
|
|
187
|
+
start(porffor, node, source, "2026-10-23"),
|
|
188
|
+
start(porffor, node, source, "2026-10-22"),
|
|
189
|
+
]);
|
|
190
|
+
|
|
191
|
+
const echo = (method: string, from: string | null, type: string | null, body: string) => ({
|
|
192
|
+
method,
|
|
193
|
+
from,
|
|
194
|
+
type,
|
|
195
|
+
body,
|
|
196
|
+
});
|
|
197
|
+
const plain = "text/plain;charset=UTF-8";
|
|
198
|
+
const bytes = new Uint8Array(256).map((_, i) => i);
|
|
199
|
+
|
|
200
|
+
// From 2026-10-23: fetch() is fetch(new Request(input, init)), as in workerd.
|
|
201
|
+
assert.deepEqual(await strict("/fetch"), {
|
|
202
|
+
viaRequest: echo("PUT", "request", plain, "req body"),
|
|
203
|
+
plusInit: echo("PATCH", "init", null, "req body"),
|
|
204
|
+
initBody: echo("PUT", null, plain, "init body"),
|
|
205
|
+
stream: echo("POST", null, null, "in a request"),
|
|
206
|
+
initStream: echo("POST", null, null, "in init"),
|
|
207
|
+
asInit: { ...echo("POST", "init-request", plain, "init req"), initUsed: true },
|
|
208
|
+
manual: 302,
|
|
209
|
+
aborted: "AbortError: The operation was aborted",
|
|
210
|
+
first: "once",
|
|
211
|
+
usedUp: true,
|
|
212
|
+
again: "TypeError: Cannot reconstruct a Request with a used body.",
|
|
213
|
+
getBody: "TypeError: Request with a GET or HEAD method cannot have a body.",
|
|
214
|
+
});
|
|
215
|
+
assert.deepEqual(await strict("/forward", { method: "POST", body: bytes, headers: { "x-from": "client" } }), {
|
|
216
|
+
copied: echo("POST", "client", null, "bytes 0-255"),
|
|
217
|
+
incomingUsed: true,
|
|
218
|
+
});
|
|
219
|
+
assert.deepEqual(await strict("/request"), {
|
|
220
|
+
copy: {
|
|
221
|
+
aUsed: true,
|
|
222
|
+
aRead:
|
|
223
|
+
"TypeError: Body has already been used. It can only be used once. Use tee() first if you need to read it twice.",
|
|
224
|
+
b: "abc",
|
|
225
|
+
headers: [["x-b", "2"]],
|
|
226
|
+
},
|
|
227
|
+
tee: ["tee me", "tee me"],
|
|
228
|
+
signal: [true, false],
|
|
229
|
+
getBody: "TypeError: Request with a GET or HEAD method cannot have a body.",
|
|
230
|
+
method: "TypeError: Invalid HTTP method string: custom",
|
|
231
|
+
redirect:
|
|
232
|
+
'TypeError: Invalid redirect value, must be one of "follow" or "manual" ("error" won\'t be implemented since it does not make sense at the edge; use "manual" and check the response status code).',
|
|
233
|
+
incoming: "TypeError: Can't modify immutable headers.",
|
|
234
|
+
fetched: "TypeError: Can't modify immutable headers.",
|
|
235
|
+
copiedResponse: "mutable",
|
|
236
|
+
incomingBody: "none",
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
// Before it: the Request's method, headers and body still reach the upstream (fixed at
|
|
240
|
+
// every date, as are stream bodies and raw bytes), but a Request can be sent twice, a
|
|
241
|
+
// copy leaves the source readable, and headers, methods and redirect modes are not checked.
|
|
242
|
+
const old = await legacy("/fetch");
|
|
243
|
+
assert.deepEqual(
|
|
244
|
+
{
|
|
245
|
+
viaRequest: old.viaRequest,
|
|
246
|
+
plusInit: old.plusInit,
|
|
247
|
+
stream: old.stream,
|
|
248
|
+
initStream: old.initStream,
|
|
249
|
+
manual: old.manual,
|
|
250
|
+
again: old.again,
|
|
251
|
+
},
|
|
252
|
+
{
|
|
253
|
+
viaRequest: echo("PUT", "request", plain, "req body"),
|
|
254
|
+
plusInit: echo("PATCH", "init", null, "req body"),
|
|
255
|
+
stream: echo("POST", null, null, "in a request"),
|
|
256
|
+
initStream: echo("POST", null, null, "in init"),
|
|
257
|
+
manual: 302,
|
|
258
|
+
again: "once",
|
|
259
|
+
},
|
|
260
|
+
);
|
|
261
|
+
assert.deepEqual(await legacy("/forward", { method: "POST", body: bytes, headers: { "x-from": "client" } }), {
|
|
262
|
+
copied: echo("POST", "client", null, "bytes 0-255"),
|
|
263
|
+
incomingUsed: false,
|
|
264
|
+
});
|
|
265
|
+
const oldRequest = await legacy("/request");
|
|
266
|
+
assert.deepEqual(
|
|
267
|
+
{
|
|
268
|
+
aUsed: oldRequest.copy.aUsed,
|
|
269
|
+
aRead: oldRequest.copy.aRead,
|
|
270
|
+
tee: oldRequest.tee,
|
|
271
|
+
signal: oldRequest.signal,
|
|
272
|
+
method: oldRequest.method,
|
|
273
|
+
redirect: oldRequest.redirect,
|
|
274
|
+
incoming: oldRequest.incoming,
|
|
275
|
+
fetched: oldRequest.fetched,
|
|
276
|
+
incomingBody: oldRequest.incomingBody,
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
aUsed: false,
|
|
280
|
+
aRead: "abc",
|
|
281
|
+
tee: ["tee me", "tee me"],
|
|
282
|
+
signal: [true, false],
|
|
283
|
+
method: "CUSTOM",
|
|
284
|
+
redirect: "error",
|
|
285
|
+
incoming: "mutable",
|
|
286
|
+
fetched: "mutable",
|
|
287
|
+
incomingBody: "object",
|
|
288
|
+
},
|
|
289
|
+
);
|
|
290
|
+
|
|
291
|
+
console.log(
|
|
292
|
+
"fetch(Request) and new Request(request, init) follow workerd from 2026-10-23 (body used up, init merging, " +
|
|
293
|
+
"GET/HEAD bodies, methods, redirect modes, signals, immutable headers); stream bodies, clone() tees and raw bytes at every date; " +
|
|
294
|
+
"2026-10-22 keeps the older Request semantics",
|
|
295
|
+
);
|
|
296
|
+
} finally {
|
|
297
|
+
for (const server of servers) {
|
|
298
|
+
server.kill(9);
|
|
299
|
+
await server.exited;
|
|
300
|
+
}
|
|
301
|
+
brokerServer.stop();
|
|
302
|
+
broker.close();
|
|
303
|
+
upstream.stop(true);
|
|
304
|
+
await rm(workdir, { recursive: true, force: true });
|
|
305
|
+
}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, readFile, readdir, rm, writeFile, stat } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { Database } from "bun:sqlite";
|
|
6
|
+
|
|
7
|
+
// Compile the exact inline C used by the embedded transport against SQLite.
|
|
8
|
+
// This tests WAL bytes, crash boundaries and fsync paths without a JS mock.
|
|
9
|
+
const transport = await readFile(new URL("../src/transport-embedded.js", import.meta.url), "utf8");
|
|
10
|
+
const start = transport.indexOf("// BEGIN LTX NATIVE");
|
|
11
|
+
const end = transport.indexOf("// END LTX NATIVE", start);
|
|
12
|
+
assert(start > 0 && end > start);
|
|
13
|
+
const work = await mkdtemp(join(tmpdir(), "sb-native-ltx-"));
|
|
14
|
+
const binary = join(work, "driver");
|
|
15
|
+
const source = `
|
|
16
|
+
#include <sqlite3.h>
|
|
17
|
+
#include <stdio.h>
|
|
18
|
+
#include <stdlib.h>
|
|
19
|
+
#include <stdint.h>
|
|
20
|
+
#include <string.h>
|
|
21
|
+
#include <strings.h>
|
|
22
|
+
#include <unistd.h>
|
|
23
|
+
#include <fcntl.h>
|
|
24
|
+
#include <sys/stat.h>
|
|
25
|
+
#include <time.h>
|
|
26
|
+
#include <signal.h>
|
|
27
|
+
#define SB_MAX_DB 16
|
|
28
|
+
${transport.slice(start, end)}
|
|
29
|
+
int main(int argc,char **argv) {
|
|
30
|
+
if(argc!=2)return 2;sb_ltx_compat=1;if(sb_ltx_register())return 3;
|
|
31
|
+
char lockpath[1024];snprintf(lockpath,sizeof(lockpath),"%s.ltx.lock",argv[1]);
|
|
32
|
+
int lock=open(lockpath,O_RDWR|O_CREAT,0600);if(lock<0||flock(lock,LOCK_EX|LOCK_NB))return 4;
|
|
33
|
+
sqlite3 *db=0;if(sqlite3_open_v2(argv[1],&db,6,"sb-ltx-bounded"))return 5;
|
|
34
|
+
if(getenv("SB_LTX_TEST_AUTO_VACUUM"))sqlite3_exec(db,"PRAGMA auto_vacuum=INCREMENTAL;",0,0,0);
|
|
35
|
+
if(sqlite3_exec(db,"PRAGMA journal_mode=WAL;PRAGMA cache_size=-2000;",0,0,0)||sb_ltx_start(0,db,argv[1],lock))return 6;
|
|
36
|
+
puts("ready");fflush(stdout);sb_ltx_test_armed=1;
|
|
37
|
+
char *line=0;size_t cap=0;
|
|
38
|
+
while(getline(&line,&cap,stdin)>0) {
|
|
39
|
+
if(strncmp(line,":break-position",15)==0){char p[1024];snprintf(p,sizeof(p),"%s.ltx/position.tmp",argv[1]);mkdir(p,0700);puts("0 injected");fflush(stdout);continue;}
|
|
40
|
+
char *error=0;int rc=sb_ltx_states[0].failed?10:sqlite3_exec(db,line,0,0,&error);
|
|
41
|
+
if(sb_ltx_after(0))rc=10;
|
|
42
|
+
printf("%d %s\\n",rc,error?error:sqlite3_errmsg(db));fflush(stdout);sqlite3_free(error);
|
|
43
|
+
}
|
|
44
|
+
free(line);sqlite3_close(db);close(lock);return 0;
|
|
45
|
+
}
|
|
46
|
+
`;
|
|
47
|
+
await writeFile(join(work, "driver.c"), source);
|
|
48
|
+
await rm(binary, { force: true });
|
|
49
|
+
const compile = Bun.spawnSync(["cc", "-O2", "-DSB_LTX_TESTING", join(work, "driver.c"), "-lsqlite3", "-o", binary]);
|
|
50
|
+
assert.equal(compile.exitCode, 0, compile.stderr.toString());
|
|
51
|
+
|
|
52
|
+
// Independent CRC baseline in Bun. This intentionally uses BigInt, unlike C.
|
|
53
|
+
const table: bigint[] = [];
|
|
54
|
+
for (let i = 0; i < 256; i++) {
|
|
55
|
+
let c = BigInt(i);
|
|
56
|
+
for (let j = 0; j < 8; j++) c = (c >> 1n) ^ (c & 1n ? 0xd800000000000000n : 0n);
|
|
57
|
+
table.push(c);
|
|
58
|
+
}
|
|
59
|
+
function crc(data: Uint8Array, init = 0xffffffffffffffffn) {
|
|
60
|
+
let c = init;
|
|
61
|
+
for (const b of data) c = table[Number((c ^ BigInt(b)) & 255n)] ^ (c >> 8n);
|
|
62
|
+
return c;
|
|
63
|
+
}
|
|
64
|
+
const flag = 0x8000000000000000n,
|
|
65
|
+
mask = 0xffffffffffffffffn;
|
|
66
|
+
function pageCrc(pgno: number, page: Uint8Array) {
|
|
67
|
+
const p = new Uint8Array(4);
|
|
68
|
+
new DataView(p.buffer).setUint32(0, pgno);
|
|
69
|
+
return flag | (crc(page, crc(p)) ^ mask);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function restore(epoch: string, output: string) {
|
|
73
|
+
const files = (await readdir(epoch)).filter((n) => n.endsWith(".ltx")).sort();
|
|
74
|
+
let db = new Uint8Array(0),
|
|
75
|
+
checksum = 0n,
|
|
76
|
+
txid = 0n;
|
|
77
|
+
for (const file of files) {
|
|
78
|
+
const b = new Uint8Array(await readFile(join(epoch, file)));
|
|
79
|
+
const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
|
|
80
|
+
assert.equal(new TextDecoder().decode(b.subarray(0, 4)), "LTX1");
|
|
81
|
+
const pageSize = v.getUint32(8),
|
|
82
|
+
commit = v.getUint32(12);
|
|
83
|
+
assert.equal(v.getBigUint64(16), txid + 1n);
|
|
84
|
+
assert.equal(v.getBigUint64(40), checksum);
|
|
85
|
+
const next = new Uint8Array(commit * pageSize);
|
|
86
|
+
next.set(db.subarray(0, next.length));
|
|
87
|
+
db = next;
|
|
88
|
+
let offset = 100,
|
|
89
|
+
hash = crc(b.subarray(0, 100));
|
|
90
|
+
const entries = [];
|
|
91
|
+
for (;;) {
|
|
92
|
+
const begin = offset,
|
|
93
|
+
pgno = v.getUint32(offset),
|
|
94
|
+
flags = v.getUint16(offset + 4);
|
|
95
|
+
hash = crc(b.subarray(offset, offset + 6), hash);
|
|
96
|
+
offset += 6;
|
|
97
|
+
if (!pgno) {
|
|
98
|
+
assert.equal(flags, 0);
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
assert.equal(flags, 1);
|
|
102
|
+
assert(pgno <= commit);
|
|
103
|
+
const size = v.getUint32(offset);
|
|
104
|
+
hash = crc(b.subarray(offset, offset + 4), hash);
|
|
105
|
+
offset += 4;
|
|
106
|
+
const stop = offset + size;
|
|
107
|
+
assert(stop <= b.length);
|
|
108
|
+
assert.equal(b[offset++], 0xf0);
|
|
109
|
+
let literals = 15,
|
|
110
|
+
byte;
|
|
111
|
+
do {
|
|
112
|
+
byte = b[offset++];
|
|
113
|
+
literals += byte;
|
|
114
|
+
} while (byte === 255);
|
|
115
|
+
assert.equal(literals, pageSize);
|
|
116
|
+
assert.equal(offset + pageSize, stop);
|
|
117
|
+
const page = b.subarray(offset, stop);
|
|
118
|
+
db.set(page, (pgno - 1) * pageSize);
|
|
119
|
+
hash = crc(page, hash);
|
|
120
|
+
offset = stop;
|
|
121
|
+
entries.push({ pgno, offset: begin, size: stop - begin });
|
|
122
|
+
}
|
|
123
|
+
const indexStart = offset;
|
|
124
|
+
function varint() {
|
|
125
|
+
let n = 0n,
|
|
126
|
+
s = 0n;
|
|
127
|
+
for (let i = 0; i < 10; i++) {
|
|
128
|
+
assert(offset < b.length);
|
|
129
|
+
const c = b[offset++];
|
|
130
|
+
n |= BigInt(c & 127) << s;
|
|
131
|
+
if (c < 128) return Number(n);
|
|
132
|
+
s += 7n;
|
|
133
|
+
}
|
|
134
|
+
throw new Error("varint overflow");
|
|
135
|
+
}
|
|
136
|
+
for (const entry of entries) {
|
|
137
|
+
assert.equal(varint(), entry.pgno);
|
|
138
|
+
assert.equal(varint(), entry.offset);
|
|
139
|
+
assert.equal(varint(), entry.size);
|
|
140
|
+
}
|
|
141
|
+
assert.equal(varint(), 0);
|
|
142
|
+
assert.equal(v.getBigUint64(offset), BigInt(offset - indexStart));
|
|
143
|
+
offset += 8;
|
|
144
|
+
hash = crc(b.subarray(indexStart, offset + 8), hash);
|
|
145
|
+
checksum = flag;
|
|
146
|
+
for (let p = 1; p <= commit; p++)
|
|
147
|
+
checksum = flag | (checksum ^ pageCrc(p, db.subarray((p - 1) * pageSize, p * pageSize)));
|
|
148
|
+
assert.equal(v.getBigUint64(offset), checksum, file);
|
|
149
|
+
assert.equal(v.getBigUint64(offset + 8), flag | (hash ^ mask), file);
|
|
150
|
+
assert.equal(offset + 16, b.length);
|
|
151
|
+
txid = v.getBigUint64(24);
|
|
152
|
+
}
|
|
153
|
+
await writeFile(output, db);
|
|
154
|
+
return { files: files.length, txid };
|
|
155
|
+
}
|
|
156
|
+
async function run(path: string, sql: string, extra: Record<string, string> = {}) {
|
|
157
|
+
const child = Bun.spawn([binary, path], {
|
|
158
|
+
env: { ...process.env, ...extra },
|
|
159
|
+
stdin: "pipe",
|
|
160
|
+
stdout: "pipe",
|
|
161
|
+
stderr: "pipe",
|
|
162
|
+
});
|
|
163
|
+
child.stdin.write(sql);
|
|
164
|
+
child.stdin.end();
|
|
165
|
+
const [code, out, error] = await Promise.all([
|
|
166
|
+
child.exited,
|
|
167
|
+
new Response(child.stdout).text(),
|
|
168
|
+
new Response(child.stderr).text(),
|
|
169
|
+
]);
|
|
170
|
+
assert.equal(code, 0, error);
|
|
171
|
+
assert(out.startsWith("ready\n"), out);
|
|
172
|
+
return out.split("\n").slice(1).filter(Boolean);
|
|
173
|
+
}
|
|
174
|
+
async function stoppedCommit(path: string, phase: string, count = 1) {
|
|
175
|
+
const child = Bun.spawn([binary, path], {
|
|
176
|
+
env: { ...process.env, SB_LTX_TEST_STOP: phase, SB_LTX_TEST_COMMIT: String(count) },
|
|
177
|
+
stdin: "pipe",
|
|
178
|
+
stdout: "pipe",
|
|
179
|
+
stderr: "pipe",
|
|
180
|
+
});
|
|
181
|
+
const reader = child.stdout.getReader();
|
|
182
|
+
let text = "";
|
|
183
|
+
async function line() {
|
|
184
|
+
while (!text.includes("\n")) {
|
|
185
|
+
const r = await reader.read();
|
|
186
|
+
if (r.done) throw new Error("driver exited before ready: " + (await new Response(child.stderr).text()));
|
|
187
|
+
text += new TextDecoder().decode(r.value);
|
|
188
|
+
}
|
|
189
|
+
const n = text.indexOf("\n");
|
|
190
|
+
const value = text.slice(0, n);
|
|
191
|
+
text = text.slice(n + 1);
|
|
192
|
+
return value;
|
|
193
|
+
}
|
|
194
|
+
assert.equal(await line(), "ready");
|
|
195
|
+
const position = await readFile(path + ".ltx/position", "utf8");
|
|
196
|
+
const epoch = position.split("\n")[1];
|
|
197
|
+
const sql =
|
|
198
|
+
count === 2
|
|
199
|
+
? "INSERT INTO events VALUES(1,'one'); INSERT INTO events VALUES(2,'two');\n"
|
|
200
|
+
: "INSERT INTO events VALUES(1,'one');\n";
|
|
201
|
+
child.stdin.write(sql);
|
|
202
|
+
child.stdin.flush();
|
|
203
|
+
let stopped = false;
|
|
204
|
+
for (let i = 0; i < 200; i++) {
|
|
205
|
+
const state = Bun.spawnSync(["ps", "-o", "state=", "-p", String(child.pid)]).stdout.toString();
|
|
206
|
+
if (state.includes("T")) {
|
|
207
|
+
stopped = true;
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
if (child.exitCode !== null) break;
|
|
211
|
+
await Bun.sleep(10);
|
|
212
|
+
}
|
|
213
|
+
child.kill("SIGKILL");
|
|
214
|
+
await child.exited;
|
|
215
|
+
assert(stopped, "did not reach crash boundary " + phase);
|
|
216
|
+
return epoch;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
try {
|
|
220
|
+
const path = join(work, "test.sqlite");
|
|
221
|
+
const sql =
|
|
222
|
+
"CREATE TABLE t(id INTEGER PRIMARY KEY,value TEXT);\n" +
|
|
223
|
+
Array.from({ length: 100 }, (_, i) => `INSERT INTO t VALUES(${i + 1},'row-${i + 1}');`).join("\n") +
|
|
224
|
+
"\n";
|
|
225
|
+
const replies = await run(path, sql);
|
|
226
|
+
assert.equal(replies.length, 101);
|
|
227
|
+
assert(
|
|
228
|
+
replies.every((n) => n.startsWith("0 ")),
|
|
229
|
+
replies.join("\n"),
|
|
230
|
+
);
|
|
231
|
+
const root = path + ".ltx";
|
|
232
|
+
const epoch = (await readdir(root)).find((n) => n.startsWith("epoch-"));
|
|
233
|
+
assert(epoch);
|
|
234
|
+
const restored = join(work, "restored.sqlite");
|
|
235
|
+
const result = await restore(join(root, epoch), restored);
|
|
236
|
+
assert.equal(result.files, 102);
|
|
237
|
+
assert.deepEqual(await readFile(restored), await readFile(path));
|
|
238
|
+
const db = new Database(restored);
|
|
239
|
+
assert.equal(db.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
|
|
240
|
+
db.close();
|
|
241
|
+
const cleanPosition = await readFile(root + "/position", "utf8");
|
|
242
|
+
assert(cleanPosition.split("\n")[2].startsWith("C "));
|
|
243
|
+
const finishedBefore = await readdir(join(root, epoch));
|
|
244
|
+
for (let n = 0; n < 5; n++) {
|
|
245
|
+
await run(path, "SELECT 1;\n");
|
|
246
|
+
assert.equal(await readFile(root + "/position", "utf8"), cleanPosition, "idle restart must not rewrite position");
|
|
247
|
+
assert.deepEqual(await readdir(join(root, epoch)), finishedBefore, "idle restart must not rewrite a base");
|
|
248
|
+
assert.equal((await readdir(root)).filter((n) => n.startsWith("epoch-")).length, 1);
|
|
249
|
+
}
|
|
250
|
+
// Size alone is not enough: an offline committed change must invalidate a clean marker.
|
|
251
|
+
const offline = new Database(path);
|
|
252
|
+
offline.exec("UPDATE t SET value='changed-offline' WHERE id=1");
|
|
253
|
+
offline.close();
|
|
254
|
+
await run(path, "SELECT 1;\n");
|
|
255
|
+
const offlineEpoch = (await readFile(root + "/position", "utf8")).split("\n")[1];
|
|
256
|
+
assert.notEqual(offlineEpoch, epoch);
|
|
257
|
+
const offlineRestore = join(work, "offline.sqlite");
|
|
258
|
+
await restore(join(root, offlineEpoch), offlineRestore);
|
|
259
|
+
assert.deepEqual(await readFile(offlineRestore), await readFile(path));
|
|
260
|
+
const bounded = join(work, "bounded.sqlite");
|
|
261
|
+
const boundedReplies = await run(
|
|
262
|
+
bounded,
|
|
263
|
+
"CREATE TABLE t(v BLOB);\nINSERT INTO t VALUES(zeroblob(1000000));\nINSERT INTO t VALUES('small');\n",
|
|
264
|
+
{ SB_LTX_MAX_WAL_BYTES: "131072" },
|
|
265
|
+
);
|
|
266
|
+
assert(boundedReplies[1].startsWith("13 "), boundedReplies.join("\n"));
|
|
267
|
+
assert(boundedReplies[2].startsWith("0 "), boundedReplies.join("\n"));
|
|
268
|
+
const live = new Database(bounded);
|
|
269
|
+
assert.equal(live.query("SELECT count(*) n FROM t").get()?.n, 1);
|
|
270
|
+
assert.equal(live.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
|
|
271
|
+
live.close();
|
|
272
|
+
const walStat = await stat(bounded + "-wal").catch(() => null);
|
|
273
|
+
assert(!walStat || walStat.size <= 131072);
|
|
274
|
+
const shrinking = join(work, "shrinking.sqlite");
|
|
275
|
+
const shrinkReplies = await run(
|
|
276
|
+
shrinking,
|
|
277
|
+
"CREATE TABLE t(v BLOB);\nBEGIN; INSERT INTO t VALUES(zeroblob(500000)); COMMIT;\nBEGIN; INSERT INTO t VALUES('rolled-back'); ROLLBACK;\nDELETE FROM t;\nPRAGMA incremental_vacuum(10000);\n",
|
|
278
|
+
{ SB_LTX_TEST_AUTO_VACUUM: "1" },
|
|
279
|
+
);
|
|
280
|
+
assert(
|
|
281
|
+
shrinkReplies.every((n) => n.startsWith("0 ")),
|
|
282
|
+
shrinkReplies.join("\n"),
|
|
283
|
+
);
|
|
284
|
+
const shrinkEpoch = (await readFile(shrinking + ".ltx/position", "utf8")).split("\n")[1];
|
|
285
|
+
const shrinkRestore = join(work, "shrinking-restored.sqlite");
|
|
286
|
+
await restore(shrinking + ".ltx/" + shrinkEpoch, shrinkRestore);
|
|
287
|
+
assert.deepEqual(await readFile(shrinking), await readFile(shrinkRestore));
|
|
288
|
+
assert((await stat(shrinking)).size < 32768, "incremental vacuum should shrink the file");
|
|
289
|
+
const failure = join(work, "failure.sqlite");
|
|
290
|
+
const failed = await run(
|
|
291
|
+
failure,
|
|
292
|
+
"CREATE TABLE events(id INTEGER PRIMARY KEY,value TEXT);\n:break-position\nINSERT INTO events VALUES(1,'committed-before-error');\nINSERT INTO events VALUES(2,'must-be-rejected');\n",
|
|
293
|
+
);
|
|
294
|
+
assert(failed[2].startsWith("10 ") && failed[3].startsWith("10 "), failed.join("\n"));
|
|
295
|
+
await rm(failure + ".ltx/position.tmp", { recursive: true });
|
|
296
|
+
await run(failure, "SELECT 1;\n");
|
|
297
|
+
const failureEpoch = (await readFile(failure + ".ltx/position", "utf8")).split("\n")[1];
|
|
298
|
+
const failureRestore = join(work, "failure-restored.sqlite");
|
|
299
|
+
await restore(failure + ".ltx/" + failureEpoch, failureRestore);
|
|
300
|
+
const failedDb = new Database(failureRestore);
|
|
301
|
+
assert.equal(failedDb.query("SELECT count(*) n FROM events").get()?.n, 1);
|
|
302
|
+
failedDb.close();
|
|
303
|
+
const phases = [
|
|
304
|
+
"wal-committed",
|
|
305
|
+
"ltx-written",
|
|
306
|
+
"ltx-fsynced",
|
|
307
|
+
"ltx-published",
|
|
308
|
+
"position-fsynced",
|
|
309
|
+
"position-published",
|
|
310
|
+
"before-checkpoint",
|
|
311
|
+
"after-checkpoint",
|
|
312
|
+
"checkpoint-record-fsynced",
|
|
313
|
+
"checkpoint-record-published",
|
|
314
|
+
];
|
|
315
|
+
for (const phase of phases) {
|
|
316
|
+
const crashPath = join(work, "crash-" + phase + ".sqlite");
|
|
317
|
+
await run(crashPath, "CREATE TABLE events(id INTEGER PRIMARY KEY,value TEXT);\n");
|
|
318
|
+
const epoch = await stoppedCommit(crashPath, phase);
|
|
319
|
+
const before = (await readdir(crashPath + ".ltx/" + epoch)).filter((n) => n.endsWith(".ltx")).length;
|
|
320
|
+
const oldRestore = join(work, "old-" + phase + ".sqlite");
|
|
321
|
+
await restore(crashPath + ".ltx/" + epoch, oldRestore);
|
|
322
|
+
const beforeDb = new Database(oldRestore);
|
|
323
|
+
assert.equal(beforeDb.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
|
|
324
|
+
const durableRows = beforeDb.query("SELECT count(*) n FROM events").get()?.n;
|
|
325
|
+
beforeDb.close();
|
|
326
|
+
assert.equal(durableRows, before > 2 ? 1 : 0);
|
|
327
|
+
await run(crashPath, "SELECT 1;\n");
|
|
328
|
+
const position = (await readFile(crashPath + ".ltx/position", "utf8")).split("\n");
|
|
329
|
+
if (phase === "position-published" || phase === "before-checkpoint" || phase === "checkpoint-record-published")
|
|
330
|
+
assert.equal(position[1], epoch, "matching WAL must resume");
|
|
331
|
+
else assert.notEqual(position[1], epoch, "mismatched position must make a fresh base");
|
|
332
|
+
const recovered = join(work, "recovered-" + phase + ".sqlite");
|
|
333
|
+
await restore(crashPath + ".ltx/" + position[1], recovered);
|
|
334
|
+
const rdb = new Database(recovered);
|
|
335
|
+
assert.equal(rdb.query("SELECT count(*) n FROM events").get()?.n, 1);
|
|
336
|
+
assert.equal(rdb.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
|
|
337
|
+
rdb.close();
|
|
338
|
+
}
|
|
339
|
+
const pending = join(work, "pending.sqlite");
|
|
340
|
+
await run(pending, "CREATE TABLE events(id INTEGER PRIMARY KEY,value TEXT);\n");
|
|
341
|
+
const pendingEpoch = await stoppedCommit(pending, "wal-committed", 2);
|
|
342
|
+
await run(pending, "SELECT 1;\n");
|
|
343
|
+
assert.equal((await readFile(pending + ".ltx/position", "utf8")).split("\n")[1], pendingEpoch);
|
|
344
|
+
const pendingRestore = join(work, "pending-restored.sqlite");
|
|
345
|
+
const pendingResult = await restore(pending + ".ltx/" + pendingEpoch, pendingRestore);
|
|
346
|
+
assert.equal(pendingResult.files, 4); // Base, schema commit and both recovered inserts.
|
|
347
|
+
const pdb = new Database(pendingRestore);
|
|
348
|
+
assert.equal(pdb.query("SELECT count(*) n FROM events").get()?.n, 2);
|
|
349
|
+
assert.equal(pdb.query("PRAGMA integrity_check").get()?.integrity_check, "ok");
|
|
350
|
+
pdb.close();
|
|
351
|
+
console.log(
|
|
352
|
+
"PASS restart: 10 SIGKILL durability boundaries, matching-prefix resume, orphan/missing/reset fallback, pending committed WAL tail captured before serving",
|
|
353
|
+
);
|
|
354
|
+
console.log(
|
|
355
|
+
"PASS native LTX: 102-file chain, independent CRC/index checks, byte-exact restore, integrity, hard WAL ceiling and oversized transaction rollback",
|
|
356
|
+
);
|
|
357
|
+
} finally {
|
|
358
|
+
await rm(work, { recursive: true, force: true });
|
|
359
|
+
}
|