@sproutboat/wire 0.20.4 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/package.json +2 -2
- package/src/broker.test.ts +53 -0
- package/src/broker.ts +66 -1
- package/src/d1.ts +232 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# @sproutboat/wire
|
|
2
2
|
|
|
3
|
+
## 0.21.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [3d576cd]
|
|
8
|
+
- @sproutboat/runtime@0.38.0
|
|
9
|
+
|
|
10
|
+
## 0.21.0
|
|
11
|
+
|
|
12
|
+
### Minor Changes
|
|
13
|
+
|
|
14
|
+
- 1f0622b: Queues parity, from compatibility date 2026-11-05 (earlier dates answer as before). The v8 content type keeps shared and circular references, array holes and invalid dates, and an Error keeps its TypeError or RangeError type (it used to throw "a circular value could not be cloned", copy a shared object, fill holes with undefined and turn an invalid Date into 1970). A message's id, timestamp, body and attempts are read-only, as in workerd. send() and sendBatch() check arguments as workerd does: no arguments, an options argument that is not an object, a sendBatch item that is not an object, `bytes` given an ArrayBuffer (workerd takes an ArrayBufferView), and a delaySeconds outside 0 to 86400, which was clamped or kept. A delivery that never comes back (the worker died or timed out) now counts as a retry: the message is delivered at most max_retries + 1 times, and then goes to the dead letter queue, where it used to be retried up to 5 times and marked dead without reaching it. The consumer announces itself to the broker (`queue.consumer`) so the broker applies this only to apps on the new date. On the embedded transport a consumer that runs past the 30 s lease renews it, so its messages are no longer delivered twice with their acks lost.
|
|
15
|
+
|
|
16
|
+
Over the broker (`sproutboat dev` and deployed apps) a queue delivery is now decoded as UTF-8 from 2026-11-05: a message body with non-ASCII text ("héllo ☃") used to reach the consumer as mojibake, because the native server hands the delivery over one character per byte.
|
|
17
|
+
|
|
18
|
+
### Patch Changes
|
|
19
|
+
|
|
20
|
+
- 4c4c25d: D1 answers like workerd in four more ways, from compatibility date 2026-11-03 (earlier dates answer exactly as before). A PRAGMA outside D1's allow-list (`table_list`, `table_info`, `table_xinfo`, `index_list`, `index_info`, `index_xinfo`, `foreign_keys`, `foreign_key_list`, `foreign_key_check`, `quick_check`, `optimize`, `defer_foreign_keys`, `case_sensitive_like`, `recursive_triggers`, `data_version`, `legacy_alter_table`, `reverse_unordered_selects`, `ignore_check_constraints`; the name as written, schema `main` only) fails with `D1_ERROR: not authorized: SQLITE_AUTH`, in a query, a batch statement or an `exec()` line, on every transport; the check runs in the shim and only looks at a statement that starts with P (or a comment or `;`). `PRAGMA foreign_keys = OFF` no longer sticks in a standalone binary (D1 runs every call in a transaction, where it is a no-op). An `exec()` error reads `... : near "SELEC": syntax error at offset 0: SQLITE_ERROR`, with the offset counted from the failing statement and SQLite's result code (the embedded `sb_sql_script` now returns both; the broker counts statement starts itself, because bun:sqlite reports the offset in the whole string). A statement with nothing in it (`;`, a comment) fails with `SQL code did not contain a statement.`, and `exec("")` with `D1_EXEC_ERROR: Error in line 1: : No SQL statements detected.`. `prepare("A; B")` runs every statement and answers with the last one (all or nothing, `meta.changes` summed, only the last statement may take parameters, a tail of comments or `;;` is "SQL code did not contain a statement."); the embedded transport does it in C only when the first statement has a non-blank tail, so a single statement costs one `strlen`. On the broker transport every whole-number parameter binds as a REAL, as in workerd (`typeof(?)` is real, `7 / 2` is 3.5, a number in a TEXT column reads "7.0"), by wrapping the placeholder in `CAST(? AS REAL)`. `meta.rows_read` is unchanged: workerd counts rows scanned with Cloudflare's SQLite fork, which upstream SQLite cannot reproduce.
|
|
21
|
+
- Updated dependencies [4c4c25d]
|
|
22
|
+
- Updated dependencies [2c19adf]
|
|
23
|
+
- Updated dependencies [354a405]
|
|
24
|
+
- Updated dependencies [1f0622b]
|
|
25
|
+
- @sproutboat/runtime@0.37.0
|
|
26
|
+
|
|
3
27
|
## 0.20.4
|
|
4
28
|
|
|
5
29
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sproutboat/wire",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -23,6 +23,6 @@
|
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
25
|
"@sproutboat/assets": "^0.3.0",
|
|
26
|
-
"@sproutboat/runtime": "^0.
|
|
26
|
+
"@sproutboat/runtime": "^0.38.0"
|
|
27
27
|
}
|
|
28
28
|
}
|
package/src/broker.test.ts
CHANGED
|
@@ -1928,3 +1928,56 @@ test("listen(): a service.fetch body streamed in FRAME_V1_MORE frames reaches th
|
|
|
1928
1928
|
socket.destroy();
|
|
1929
1929
|
server.stop();
|
|
1930
1930
|
});
|
|
1931
|
+
|
|
1932
|
+
test("Queues 2026-11-05: a delivery that never comes back is a retry, and past max_retries goes to the dead letter queue", async () => {
|
|
1933
|
+
const delivered: JsonObject[] = [];
|
|
1934
|
+
const fetchImpl: FetchLike = async (_url, init) => {
|
|
1935
|
+
delivered.push(obj(parseJsonValue(String(init?.body))));
|
|
1936
|
+
throw new Error("the sprout died");
|
|
1937
|
+
};
|
|
1938
|
+
const b = make({
|
|
1939
|
+
bindings: {
|
|
1940
|
+
queues: ["JOBS", "DEAD"],
|
|
1941
|
+
queueConfig: [
|
|
1942
|
+
{ binding: "JOBS", queue: "jobs", maxRetries: 1, deadLetterQueue: "dead" },
|
|
1943
|
+
{ binding: "DEAD", queue: "dead" },
|
|
1944
|
+
],
|
|
1945
|
+
},
|
|
1946
|
+
sproutUrl: "http://127.0.0.1:1/",
|
|
1947
|
+
fetchImpl,
|
|
1948
|
+
deliveryTimeoutMs: 100,
|
|
1949
|
+
deliveryLeaseMs: 200,
|
|
1950
|
+
});
|
|
1951
|
+
await b.dispatch({ op: "queue.consumer", queue: "JOBS" });
|
|
1952
|
+
await b.dispatch({ op: "queue.send", queue: "JOBS", body: "work" });
|
|
1953
|
+
await Bun.sleep(2500);
|
|
1954
|
+
// max_retries 1: two deliveries, then the dead letter queue (whose consumer is this sprout too).
|
|
1955
|
+
expect(delivered.filter((d) => d.queue === "JOBS").length).toBe(2);
|
|
1956
|
+
expect(delivered.filter((d) => d.queue === "DEAD").length).toBeGreaterThan(0);
|
|
1957
|
+
expect(await b.dispatch({ op: "queue.metrics", queue: "JOBS" })).toMatchObject({ metrics: { backlogCount: 0 } });
|
|
1958
|
+
});
|
|
1959
|
+
|
|
1960
|
+
test("Queues: without the 2026-11-05 announcement a lost delivery is retried after 5 s, as before", async () => {
|
|
1961
|
+
const delivered: JsonObject[] = [];
|
|
1962
|
+
const fetchImpl: FetchLike = async (_url, init) => {
|
|
1963
|
+
delivered.push(obj(parseJsonValue(String(init?.body))));
|
|
1964
|
+
throw new Error("the sprout died");
|
|
1965
|
+
};
|
|
1966
|
+
const b = make({
|
|
1967
|
+
bindings: {
|
|
1968
|
+
queues: ["JOBS", "DEAD"],
|
|
1969
|
+
queueConfig: [
|
|
1970
|
+
{ binding: "JOBS", queue: "jobs", maxRetries: 1, deadLetterQueue: "dead" },
|
|
1971
|
+
{ binding: "DEAD", queue: "dead" },
|
|
1972
|
+
],
|
|
1973
|
+
},
|
|
1974
|
+
sproutUrl: "http://127.0.0.1:1/",
|
|
1975
|
+
fetchImpl,
|
|
1976
|
+
deliveryTimeoutMs: 100,
|
|
1977
|
+
deliveryLeaseMs: 200,
|
|
1978
|
+
});
|
|
1979
|
+
await b.dispatch({ op: "queue.send", queue: "JOBS", body: "work" });
|
|
1980
|
+
await Bun.sleep(1500);
|
|
1981
|
+
expect(delivered.filter((d) => d.queue === "DEAD").length).toBe(0);
|
|
1982
|
+
expect(delivered.filter((d) => d.queue === "JOBS").length).toBe(1);
|
|
1983
|
+
});
|
package/src/broker.ts
CHANGED
|
@@ -56,6 +56,9 @@ export type Bindings = {
|
|
|
56
56
|
maxBatchSize?: number;
|
|
57
57
|
maxBatchTimeout?: number;
|
|
58
58
|
maxRetries?: number;
|
|
59
|
+
retryDelay?: number;
|
|
60
|
+
/** The queue name of this queue's dead letter queue. */
|
|
61
|
+
deadLetterQueue?: string;
|
|
59
62
|
}>;
|
|
60
63
|
analytics: string[];
|
|
61
64
|
do: Array<{ binding: string; className: string }>;
|
|
@@ -678,6 +681,16 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
678
681
|
const queueStore = (part: string): { store: Database; part: string } =>
|
|
679
682
|
part.startsWith("#") ? { store: db, part } : storeFor("queue", requireQueue(part));
|
|
680
683
|
const queueSettingsOf = (binding: string) => bindings.queueConfig?.find((q) => q.binding === binding);
|
|
684
|
+
/** Queue bindings whose sprout follows the 2026-11-05 delivery rules (see "queue.consumer"). */
|
|
685
|
+
const strictQueues = new Set<string>();
|
|
686
|
+
/** The partition of a queue binding's dead letter queue: its binding when the sprout binds that
|
|
687
|
+
* queue, else "#name" (as the sprout computes it), or "" when there is none. */
|
|
688
|
+
const deadLetterPartOf = (binding: string): string => {
|
|
689
|
+
const name = queueSettingsOf(binding)?.deadLetterQueue;
|
|
690
|
+
if (name === undefined) return "";
|
|
691
|
+
const bound = bindings.queues.find((b) => (queueSettingsOf(b)?.queue ?? b) === name);
|
|
692
|
+
return bound ?? "#" + name;
|
|
693
|
+
};
|
|
681
694
|
const requireAe = bound(bindings.analytics, "analytics dataset");
|
|
682
695
|
const rateLimiterNames = bindings.ratelimiters.map((r) => r.binding);
|
|
683
696
|
const requireRateLimiter = bound(rateLimiterNames, "rate limiter");
|
|
@@ -1801,6 +1814,12 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
1801
1814
|
? { ok: true, count: msgs.length, metrics: queueBacklog(store, q) }
|
|
1802
1815
|
: { ok: true, count: msgs.length };
|
|
1803
1816
|
}
|
|
1817
|
+
case "queue.consumer": {
|
|
1818
|
+
// A sprout from compatibility date 2026-11-05 announces its queue before its handler
|
|
1819
|
+
// runs: from then on a delivery that never comes back counts as a retry.
|
|
1820
|
+
strictQueues.add(requireQueue(msg.queue));
|
|
1821
|
+
return { ok: true };
|
|
1822
|
+
}
|
|
1804
1823
|
case "queue.metrics": {
|
|
1805
1824
|
const { store, part: q } = storeFor("queue", requireQueue(msg.queue));
|
|
1806
1825
|
return { ok: true, metrics: queueBacklog(store, q) };
|
|
@@ -2256,11 +2275,44 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2256
2275
|
const wait = Math.max(0, Number(settings?.maxBatchTimeout) || 0) * 1000;
|
|
2257
2276
|
// Deliveries that never came back (the sprout died or timed out each time) stop after
|
|
2258
2277
|
// 5 attempts, or max_retries + 1 when that is more.
|
|
2259
|
-
const
|
|
2278
|
+
const strict = strictQueues.has(binding);
|
|
2279
|
+
const maxRetries = settings?.maxRetries ?? 3;
|
|
2280
|
+
// From 2026-11-05 a lost delivery counts as a retry, so the cap is exactly max_retries + 1
|
|
2281
|
+
// and the message goes to the dead letter queue like any other past it.
|
|
2282
|
+
const cap = strict ? maxRetries + 1 : Math.max(QUEUE_MAX_ATTEMPTS, maxRetries + 1);
|
|
2283
|
+
const deadPart = strict ? deadLetterPartOf(binding) : "";
|
|
2260
2284
|
const now = Date.now();
|
|
2261
2285
|
const claim = randomBytes(24).toString("hex");
|
|
2262
2286
|
const rows = store
|
|
2263
2287
|
.transaction(() => {
|
|
2288
|
+
const lost = strict
|
|
2289
|
+
? store
|
|
2290
|
+
.query<{ id: string; body: string; sent_at: number }, [string, number, number]>(
|
|
2291
|
+
"SELECT id, body, sent_at FROM mq WHERE queue = ? AND dead = 0 AND visible_at <= ? AND attempts >= ?",
|
|
2292
|
+
)
|
|
2293
|
+
.all(part, now, cap)
|
|
2294
|
+
: [];
|
|
2295
|
+
if (lost.length && deadPart) {
|
|
2296
|
+
try {
|
|
2297
|
+
const target = queueStore(deadPart);
|
|
2298
|
+
for (const r of lost) {
|
|
2299
|
+
if (target.store === store)
|
|
2300
|
+
store
|
|
2301
|
+
.query(
|
|
2302
|
+
"UPDATE mq SET queue = ?, visible_at = ?, attempts = 0, claim_token = '', dead = 0 WHERE queue = ? AND id = ?",
|
|
2303
|
+
)
|
|
2304
|
+
.run(target.part, now, part, r.id);
|
|
2305
|
+
else {
|
|
2306
|
+
target.store
|
|
2307
|
+
.query("INSERT OR IGNORE INTO mq (queue, id, body, visible_at, sent_at) VALUES (?, ?, ?, ?, ?)")
|
|
2308
|
+
.run(target.part, r.id, r.body, now, r.sent_at);
|
|
2309
|
+
store.query("DELETE FROM mq WHERE queue = ? AND id = ?").run(part, r.id);
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
} catch {
|
|
2313
|
+
// not a queue this broker serves: they are marked dead below, like a queue without one
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2264
2316
|
store
|
|
2265
2317
|
.query("UPDATE mq SET dead = 1 WHERE queue = ? AND visible_at <= ? AND attempts >= ?")
|
|
2266
2318
|
.run(part, now, cap);
|
|
@@ -2302,6 +2354,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2302
2354
|
let dead: string[] = [];
|
|
2303
2355
|
let deadLetter = "";
|
|
2304
2356
|
let delays: JsonObject | null = null;
|
|
2357
|
+
let answered = false;
|
|
2305
2358
|
if (res?.ok) {
|
|
2306
2359
|
try {
|
|
2307
2360
|
const parsed = jsonObject(parseJsonValue(await res.text()));
|
|
@@ -2314,6 +2367,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2314
2367
|
dead = (parsed && ids(parsed.dead)) ?? [];
|
|
2315
2368
|
deadLetter = parsed && isString(parsed.deadLetter) ? parsed.deadLetter : "";
|
|
2316
2369
|
delays = (parsed && jsonObject(parsed.delays ?? null)) ?? null;
|
|
2370
|
+
answered = true;
|
|
2317
2371
|
} catch {
|
|
2318
2372
|
ack = [];
|
|
2319
2373
|
retry = rows.map((r) => r.id);
|
|
@@ -2322,6 +2376,17 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
|
|
|
2322
2376
|
ack = [];
|
|
2323
2377
|
retry = rows.map((r) => r.id);
|
|
2324
2378
|
}
|
|
2379
|
+
if (strict && !answered) {
|
|
2380
|
+
// No usable answer (the sprout timed out, died or sent garbage): from 2026-11-05 that is
|
|
2381
|
+
// a retry like the sprout's own, with the queue's retry delay, and a message on its last
|
|
2382
|
+
// allowed delivery goes to the dead letter queue.
|
|
2383
|
+
const seconds = Math.max(0, Number(settings?.retryDelay) || 0);
|
|
2384
|
+
const last = new Set(rows.filter((r) => r.attempts + 1 > maxRetries).map((r) => r.id));
|
|
2385
|
+
dead = retry.filter((id) => last.has(id));
|
|
2386
|
+
deadLetter = deadPart;
|
|
2387
|
+
delays = Object.fromEntries(retry.filter((id) => !last.has(id)).map((id) => [id, seconds]));
|
|
2388
|
+
retry = retry.filter((id) => !last.has(id));
|
|
2389
|
+
}
|
|
2325
2390
|
if (closed || !dispatchEnabled()) return;
|
|
2326
2391
|
const retrySet = new Set(retry);
|
|
2327
2392
|
const deadSet = new Set(dead);
|
package/src/d1.ts
CHANGED
|
@@ -64,10 +64,11 @@ const SQLITE_CODES = [
|
|
|
64
64
|
"SQLITE_NOTADB",
|
|
65
65
|
];
|
|
66
66
|
/** D1's wording for a SQLite error, as the embedded transport words it for strict calls. */
|
|
67
|
-
export function d1Error(error: Error): Error {
|
|
67
|
+
export function d1Error(error: Error, byteOffset?: number): Error {
|
|
68
68
|
if (!isSqliteFailure(error)) return error;
|
|
69
69
|
let text = `D1_ERROR: ${error.message}`;
|
|
70
|
-
|
|
70
|
+
const offset = byteOffset ?? error.byteOffset;
|
|
71
|
+
if (Number.isInteger(offset) && Number(offset) >= 0) text += ` at offset ${offset}`;
|
|
71
72
|
const base = SQLITE_CODES[(Number(error.errno) || 0) & 255];
|
|
72
73
|
if (base) {
|
|
73
74
|
text += `: ${base}`;
|
|
@@ -75,6 +76,160 @@ export function d1Error(error: Error): Error {
|
|
|
75
76
|
}
|
|
76
77
|
return new Error(text);
|
|
77
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Where, in a line of statements, the statement that fails at byte `offset` begins. bun:sqlite counts an
|
|
81
|
+
* error's offset from the start of the whole string; D1 (SQLite's own exec loop) counts it from the first
|
|
82
|
+
* non-space character of the failing statement. A `;` inside a quoted name or string or a comment does not
|
|
83
|
+
* end a statement.
|
|
84
|
+
*/
|
|
85
|
+
export function statementStart(sql: Uint8Array, offset: number): number {
|
|
86
|
+
const at = (i: number) => (i < sql.length ? sql[i] : 0);
|
|
87
|
+
const space = (c: number) => c === 32 || c === 9 || c === 10 || c === 13 || c === 12;
|
|
88
|
+
let start = 0;
|
|
89
|
+
let i = 0;
|
|
90
|
+
while (i < offset) {
|
|
91
|
+
const c = sql[i];
|
|
92
|
+
if (c === 59) {
|
|
93
|
+
i++;
|
|
94
|
+
while (space(at(i))) i++;
|
|
95
|
+
if (i <= offset) start = i;
|
|
96
|
+
} else if (c === 39 || c === 34 || c === 96 || c === 91) {
|
|
97
|
+
const close = c === 91 ? 93 : c;
|
|
98
|
+
i++;
|
|
99
|
+
while (i < sql.length && sql[i] !== close) i++;
|
|
100
|
+
i++;
|
|
101
|
+
} else if (c === 45 && at(i + 1) === 45) {
|
|
102
|
+
while (i < sql.length && sql[i] !== 10) i++;
|
|
103
|
+
} else if (c === 47 && at(i + 1) === 42) {
|
|
104
|
+
i += 2;
|
|
105
|
+
while (i < sql.length && !(sql[i] === 42 && at(i + 1) === 47)) i++;
|
|
106
|
+
i += 2;
|
|
107
|
+
} else i++;
|
|
108
|
+
}
|
|
109
|
+
return start;
|
|
110
|
+
}
|
|
111
|
+
/** D1's text for the failure of one exec line (from 2026-11-03: with " at offset N: SQLITE_ERROR" and all). */
|
|
112
|
+
function execLineError(error: Error, line: string): string {
|
|
113
|
+
const failure = error;
|
|
114
|
+
if (!isSqliteFailure(failure)) return failure.message;
|
|
115
|
+
const offset = Number(failure.byteOffset);
|
|
116
|
+
const own =
|
|
117
|
+
Number.isInteger(offset) && offset > 0
|
|
118
|
+
? offset - statementStart(new TextEncoder().encode(line), offset)
|
|
119
|
+
: undefined;
|
|
120
|
+
return d1Error(failure, own).message.replace(/^D1_ERROR: /, "");
|
|
121
|
+
}
|
|
122
|
+
/** A parameter in SQL text: where it sits, and which bound value it takes. */
|
|
123
|
+
type Placeholder = { start: number; end: number; index: number };
|
|
124
|
+
const placeholderCache = new Map<string, Placeholder[]>();
|
|
125
|
+
const isWordChar = (c: number) =>
|
|
126
|
+
(c >= 48 && c <= 57) || (c >= 65 && c <= 90) || c === 95 || (c >= 97 && c <= 122) || c > 127;
|
|
127
|
+
/** The position of the `;` that ends the first statement of `sql` (strings, quoted names and comments skipped), or -1. */
|
|
128
|
+
export function firstSemicolon(sql: string): number {
|
|
129
|
+
for (let i = 0; i < sql.length; i++) {
|
|
130
|
+
const c = sql.charCodeAt(i);
|
|
131
|
+
if (c === 59) return i;
|
|
132
|
+
if (c === 39 || c === 34 || c === 96 || c === 91) {
|
|
133
|
+
const close = sql.indexOf(c === 91 ? "]" : sql[i]!, i + 1);
|
|
134
|
+
if (close < 0) return -1;
|
|
135
|
+
i = close;
|
|
136
|
+
} else if (c === 45 && sql.charCodeAt(i + 1) === 45) {
|
|
137
|
+
const nl = sql.indexOf("\n", i);
|
|
138
|
+
if (nl < 0) return -1;
|
|
139
|
+
i = nl;
|
|
140
|
+
} else if (c === 47 && sql.charCodeAt(i + 1) === 42) {
|
|
141
|
+
const close = sql.indexOf("*/", i + 2);
|
|
142
|
+
if (close < 0) return -1;
|
|
143
|
+
i = close + 1;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return -1;
|
|
147
|
+
}
|
|
148
|
+
function placeholders(sql: string): Placeholder[] {
|
|
149
|
+
const known = placeholderCache.get(sql);
|
|
150
|
+
if (known) return known;
|
|
151
|
+
const found: Placeholder[] = [];
|
|
152
|
+
const names = new Map<string, number>();
|
|
153
|
+
let max = 0;
|
|
154
|
+
for (let i = 0; i < sql.length; i++) {
|
|
155
|
+
const c = sql.charCodeAt(i);
|
|
156
|
+
if (c === 39 || c === 34 || c === 96 || c === 91) {
|
|
157
|
+
const close = sql.indexOf(c === 91 ? "]" : sql[i]!, i + 1);
|
|
158
|
+
if (close < 0) break;
|
|
159
|
+
i = close;
|
|
160
|
+
} else if (c === 45 && sql.charCodeAt(i + 1) === 45) {
|
|
161
|
+
const nl = sql.indexOf("\n", i);
|
|
162
|
+
if (nl < 0) break;
|
|
163
|
+
i = nl;
|
|
164
|
+
} else if (c === 47 && sql.charCodeAt(i + 1) === 42) {
|
|
165
|
+
const close = sql.indexOf("*/", i + 2);
|
|
166
|
+
if (close < 0) break;
|
|
167
|
+
i = close + 1;
|
|
168
|
+
} else if (c === 63) {
|
|
169
|
+
let end = i + 1;
|
|
170
|
+
while (end < sql.length && sql.charCodeAt(end) >= 48 && sql.charCodeAt(end) <= 57) end++;
|
|
171
|
+
const index = end > i + 1 ? Number(sql.slice(i + 1, end)) : max + 1;
|
|
172
|
+
max = Math.max(max, index);
|
|
173
|
+
found.push({ start: i, end, index });
|
|
174
|
+
i = end - 1;
|
|
175
|
+
} else if (c === 58 || c === 64 || c === 36) {
|
|
176
|
+
let end = i + 1;
|
|
177
|
+
while (end < sql.length && isWordChar(sql.charCodeAt(end))) end++;
|
|
178
|
+
if (end > i + 1) {
|
|
179
|
+
const name = sql.slice(i, end);
|
|
180
|
+
let index = names.get(name);
|
|
181
|
+
if (index === undefined) {
|
|
182
|
+
index = ++max;
|
|
183
|
+
names.set(name, index);
|
|
184
|
+
}
|
|
185
|
+
found.push({ start: i, end, index });
|
|
186
|
+
i = end - 1;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (placeholderCache.size >= 256) {
|
|
191
|
+
const oldest = placeholderCache.keys().next();
|
|
192
|
+
if (!oldest.done) placeholderCache.delete(oldest.value);
|
|
193
|
+
}
|
|
194
|
+
placeholderCache.set(sql, found);
|
|
195
|
+
return found;
|
|
196
|
+
}
|
|
197
|
+
const REAL_PREFIX = "CAST(";
|
|
198
|
+
const REAL_SUFFIX = " AS REAL)";
|
|
199
|
+
/**
|
|
200
|
+
* workerd binds every JS number (and boolean) as a REAL; bun:sqlite binds a whole number as an INTEGER, and
|
|
201
|
+
* has no way to say otherwise. Wrap the placeholders that take one: `CAST(?1 AS REAL)` is the same value as
|
|
202
|
+
* the REAL SQLite would have been given, keeps an index lookup (`WHERE id = CAST(? AS REAL)` still searches
|
|
203
|
+
* the rowid) and reads `typeof(?)` as real, `7 / 2` as 3.5, and a number stored in a TEXT column as "7.0".
|
|
204
|
+
* `wraps` are the start offsets of the wrapped placeholders in the new text, to put an error offset back.
|
|
205
|
+
*/
|
|
206
|
+
export function bindReal(sql: string, values: SqlParam[]): { sql: string; wraps: number[] } | undefined {
|
|
207
|
+
let out: string | undefined;
|
|
208
|
+
let last = 0;
|
|
209
|
+
const wraps: number[] = [];
|
|
210
|
+
for (const p of placeholders(sql)) {
|
|
211
|
+
const v = values[p.index - 1];
|
|
212
|
+
if (!Number.isInteger(v)) continue;
|
|
213
|
+
out = (out ?? "") + sql.slice(last, p.start);
|
|
214
|
+
wraps.push(out.length);
|
|
215
|
+
out += REAL_PREFIX + sql.slice(p.start, p.end) + REAL_SUFFIX;
|
|
216
|
+
last = p.end;
|
|
217
|
+
}
|
|
218
|
+
return out === undefined ? undefined : { sql: out + sql.slice(last), wraps };
|
|
219
|
+
}
|
|
220
|
+
/** An error offset in bindReal's text, in the text the caller wrote. */
|
|
221
|
+
function unwrapOffset(text: string, wraps: number[], offset: number): number {
|
|
222
|
+
let shift = 0;
|
|
223
|
+
for (const at of wraps) {
|
|
224
|
+
const start = Buffer.byteLength(text.slice(0, at));
|
|
225
|
+
if (offset < start) break;
|
|
226
|
+
shift += Math.min(offset - start, REAL_PREFIX.length);
|
|
227
|
+
const body = text.indexOf(" AS REAL)", at) - (at + REAL_PREFIX.length);
|
|
228
|
+
if (offset >= start + REAL_PREFIX.length + body)
|
|
229
|
+
shift += Math.min(offset - (start + REAL_PREFIX.length + body), REAL_SUFFIX.length);
|
|
230
|
+
}
|
|
231
|
+
return offset - shift;
|
|
232
|
+
}
|
|
78
233
|
export type D1Request = { owner: string; key: string; path: string; name: string; msg: Frame };
|
|
79
234
|
|
|
80
235
|
/** maxOpen > 0 (Durable Object SQL, one file per object): at most that many connections stay
|
|
@@ -87,22 +242,79 @@ export function createD1Store(executionMs = 0, capability = "", maxOpen = 0) {
|
|
|
87
242
|
// and rows come back as arrays with their column names.
|
|
88
243
|
// A read changes nothing: changes() still holds the last write's count, total_changes() does not move.
|
|
89
244
|
const total = (db: Database) => db.query<{ n: number }, []>("SELECT total_changes() AS n").get()?.n ?? 0;
|
|
90
|
-
|
|
245
|
+
type Statement = ReturnType<Database["query"]>;
|
|
246
|
+
// The text after statement's own (a statement bun:sqlite prepared; it ignores what follows): blank when it is
|
|
247
|
+
// the only one. Unbound, a statement's text is its own source; one with parameters can only be the last.
|
|
248
|
+
const tailOf = (statement: Statement, text: string) => {
|
|
249
|
+
const semi = text.indexOf(";");
|
|
250
|
+
if (semi < 0 || text.slice(semi + 1).trim() === "") return "";
|
|
251
|
+
let own = text;
|
|
252
|
+
if (statement.paramsCount === 0) own = statement.toString();
|
|
253
|
+
else {
|
|
254
|
+
const end = firstSemicolon(text);
|
|
255
|
+
if (end >= 0) own = text.slice(0, end + 1);
|
|
256
|
+
}
|
|
257
|
+
return text.startsWith(own) ? text.slice(own.length) : "";
|
|
258
|
+
};
|
|
259
|
+
const runStrict = (db: Database, sql: string, values: SqlParam[], parity = false) => {
|
|
91
260
|
const started = performance.now();
|
|
92
261
|
const before = total(db);
|
|
262
|
+
// 2026-11-03: numbers bind as REAL, as in workerd (see bindReal)
|
|
263
|
+
const real = parity ? bindReal(sql, values) : undefined;
|
|
264
|
+
const text = real ? real.sql : sql;
|
|
265
|
+
const fail = (e: Error) => {
|
|
266
|
+
const at = Number(isSqliteFailure(e) ? e.byteOffset : -1);
|
|
267
|
+
return d1Error(e, real && at >= 0 ? unwrapOffset(text, real.wraps, at) : undefined);
|
|
268
|
+
};
|
|
93
269
|
let statement;
|
|
94
270
|
try {
|
|
95
|
-
statement = db.query<SqliteValue[], SqlParam[]>(
|
|
271
|
+
statement = db.query<SqliteValue[], SqlParam[]>(text);
|
|
96
272
|
} catch (error) {
|
|
97
|
-
throw
|
|
273
|
+
throw fail(error instanceof Error ? error : new Error(String(error)));
|
|
274
|
+
}
|
|
275
|
+
// 2026-11-03: a statement with more behind it runs them all and answers with the last, as workerd
|
|
276
|
+
// does (one savepoint: all or none), and only the last can take parameters.
|
|
277
|
+
const rest = parity ? tailOf(statement, text) : "";
|
|
278
|
+
let multi = false;
|
|
279
|
+
if (rest.trim() !== "") {
|
|
280
|
+
multi = true;
|
|
281
|
+
db.exec("SAVEPOINT sb_d1_multi");
|
|
98
282
|
}
|
|
99
|
-
if (statement.paramsCount !== values.length)
|
|
100
|
-
throw new Error("D1_ERROR: Wrong number of parameter bindings for SQL query.");
|
|
101
283
|
let rows: SqliteValue[][];
|
|
102
284
|
try {
|
|
285
|
+
if (multi) {
|
|
286
|
+
let more = rest;
|
|
287
|
+
for (;;) {
|
|
288
|
+
more = more.trimStart();
|
|
289
|
+
if (more === "") break;
|
|
290
|
+
if (statement.paramsCount > 0)
|
|
291
|
+
throw new Error(
|
|
292
|
+
"D1_ERROR: When executing multiple SQL statements in a single call, only the last statement can have parameters.",
|
|
293
|
+
);
|
|
294
|
+
statement.run();
|
|
295
|
+
let next;
|
|
296
|
+
try {
|
|
297
|
+
next = db.query<SqliteValue[], SqlParam[]>(more);
|
|
298
|
+
} catch (error) {
|
|
299
|
+
// bun:sqlite has no statement to give for a tail of comments or semicolons
|
|
300
|
+
if (error instanceof Error && !isSqliteFailure(error))
|
|
301
|
+
throw new Error("D1_ERROR: SQL code did not contain a statement.");
|
|
302
|
+
throw error;
|
|
303
|
+
}
|
|
304
|
+
statement = next;
|
|
305
|
+
more = tailOf(next, more);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (statement.paramsCount !== values.length)
|
|
309
|
+
throw new Error("D1_ERROR: Wrong number of parameter bindings for SQL query.");
|
|
103
310
|
rows = statement.values(...values) ?? [];
|
|
311
|
+
if (multi) db.exec("RELEASE sb_d1_multi");
|
|
104
312
|
} catch (error) {
|
|
105
|
-
|
|
313
|
+
if (multi) {
|
|
314
|
+
db.exec("ROLLBACK TO sb_d1_multi");
|
|
315
|
+
db.exec("RELEASE sb_d1_multi");
|
|
316
|
+
}
|
|
317
|
+
throw fail(error instanceof Error ? error : new Error(String(error)));
|
|
106
318
|
}
|
|
107
319
|
const meta = db
|
|
108
320
|
.query<{ changes: number; last_row_id: number }, []>(
|
|
@@ -114,7 +326,7 @@ export function createD1Store(executionMs = 0, capability = "", maxOpen = 0) {
|
|
|
114
326
|
rows: rows.map((row) => row.map((v) => cell(v, true))),
|
|
115
327
|
meta: {
|
|
116
328
|
duration: performance.now() - started,
|
|
117
|
-
changes: total(db) > before ? (meta?.changes ?? 0) : 0,
|
|
329
|
+
changes: multi ? total(db) - before : total(db) > before ? (meta?.changes ?? 0) : 0,
|
|
118
330
|
last_row_id: meta?.last_row_id ?? 0,
|
|
119
331
|
rows_read: rows.length,
|
|
120
332
|
},
|
|
@@ -199,7 +411,10 @@ export function createD1Store(executionMs = 0, capability = "", maxOpen = 0) {
|
|
|
199
411
|
// Foreign keys are enforced from compatibility date 2026-10-24, as in D1. The
|
|
200
412
|
// pragma is per connection and a no-op inside a transaction, so set it per call.
|
|
201
413
|
if (msg.op !== "d1.backup") db.exec(strict ? "PRAGMA foreign_keys = ON" : "PRAGMA foreign_keys = OFF");
|
|
202
|
-
const
|
|
414
|
+
const parity = msg.parity === true;
|
|
415
|
+
const exec1 = strict
|
|
416
|
+
? (d: Database, sql: string, values: SqlParam[]) => runStrict(d, sql, values, parity)
|
|
417
|
+
: run;
|
|
203
418
|
switch (msg.op) {
|
|
204
419
|
case "d1.query":
|
|
205
420
|
return { ok: true, ...exec1(db, str(msg.sql), params(msg.params)) };
|
|
@@ -216,7 +431,13 @@ export function createD1Store(executionMs = 0, capability = "", maxOpen = 0) {
|
|
|
216
431
|
db.exec(line);
|
|
217
432
|
} catch (error) {
|
|
218
433
|
throw new Error(
|
|
219
|
-
`D1_EXEC_ERROR: Error in line ${i + 1}: ${line}: ${
|
|
434
|
+
`D1_EXEC_ERROR: Error in line ${i + 1}: ${line}: ${
|
|
435
|
+
msg.parity === true
|
|
436
|
+
? execLineError(error instanceof Error ? error : new Error(String(error)), line)
|
|
437
|
+
: error instanceof Error
|
|
438
|
+
? error.message
|
|
439
|
+
: String(error)
|
|
440
|
+
}`,
|
|
220
441
|
);
|
|
221
442
|
}
|
|
222
443
|
}),
|