cursedbelt-server 4.15.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/d1/http.d.ts +66 -0
- package/dist/server/d1/http.js +143 -0
- package/dist/server/d1/index.d.ts +1 -0
- package/dist/server/d1/index.js +1 -0
- package/dist/server/d1/localBackup.d.ts +1 -0
- package/dist/server/d1/localBackup.js +4 -0
- package/dist/server/d1/pullD1.d.ts +90 -0
- package/dist/server/d1/pullD1.js +149 -0
- package/package.json +1 -1
- package/src/server/d1/http.spec.ts +126 -0
- package/src/server/d1/http.ts +182 -0
- package/src/server/d1/index.ts +1 -0
- package/src/server/d1/localBackup.ts +14 -0
- package/src/server/d1/pullD1.spec.ts +166 -0
- package/src/server/d1/pullD1.ts +184 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A D1 database reached over Cloudflare's REST API — for a process that is NOT a Worker.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this exists ─────────────────────────────────────────────────────────────────────
|
|
5
|
+
* An app that moves onto a Worker keeps some work on the Mac: a job that drives the owner's
|
|
6
|
+
* own Chrome, a bulk importer reading a local folder, a backfill, a backup. Before the move
|
|
7
|
+
* those wrote the Mac's `bun:sqlite` file; after it, that file is a frozen copy nothing reads,
|
|
8
|
+
* and a script writing into it reports success while the library it meant to change never
|
|
9
|
+
* hears about it. Measured on `music`'s cutover plan, 2026-09-22: four Mac-side writers, and
|
|
10
|
+
* the only other door into D1 was a per-app bearer route on the Worker for each one.
|
|
11
|
+
*
|
|
12
|
+
* So the Mac gets the same {@link D1LikeDatabase} a Worker holds, pointed at the same
|
|
13
|
+
* database, and the app's own catalogue code runs unchanged against production — ONE writer
|
|
14
|
+
* of the rows, one schema, one set of statements.
|
|
15
|
+
*
|
|
16
|
+
* ── How ──────────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
* It is a {@link D1BindingLike} over `POST /accounts/:account/d1/database/:db/query` (and
|
|
18
|
+
* `/raw`), handed to {@link createRemoteD1} — so every value normalisation, limit check and
|
|
19
|
+
* `success: false` guard is the binding driver's own, not a second copy that could drift.
|
|
20
|
+
* `batch()` sends `{ batch: [...] }`, which D1 runs as one unit exactly as a binding's batch
|
|
21
|
+
* does: measured 2026-09-23 on a throwaway database, a batch whose second statement violated
|
|
22
|
+
* a UNIQUE constraint left the first one's row absent.
|
|
23
|
+
*
|
|
24
|
+
* 🔴 **Not for a Worker.** A Worker has the binding, which is faster, free of an API token,
|
|
25
|
+
* and not subject to the REST API's rate limit. This is for a Mac process, with the account
|
|
26
|
+
* token read from the generation's secrets — never shipped to a browser or a Worker.
|
|
27
|
+
*
|
|
28
|
+
* 🔴 **Every number arrives as a REAL.** The REST API's parameters are JSON, and measured
|
|
29
|
+
* 2026-09-23 it binds a JSON `5` as the REAL `5.0` (`typeof(?)` answers `real`). Everywhere a
|
|
30
|
+
* number is compared, added, used as a `LIMIT`, or stored in an INTEGER column, that is
|
|
31
|
+
* indistinguishable from an integer — SQLite compares `5.0 = 5` true and INTEGER affinity
|
|
32
|
+
* stores `5`. The one place it differs: a number stored into a TEXT column is `'5.0'`, where
|
|
33
|
+
* the binding stores `'5'`. Bind a string into a TEXT column. (Sending integers as strings
|
|
34
|
+
* instead was measured too, and is worse: it breaks every comparison that has no column
|
|
35
|
+
* affinity, such as `CASE WHEN ? = 1`.)
|
|
36
|
+
*
|
|
37
|
+
* 🔴 **A BLOB parameter is refused.** The REST API's parameters are JSON; there is no
|
|
38
|
+
* lossless spelling of bytes in it that D1 documents, and guessing one would store the
|
|
39
|
+
* wrong thing silently. Encode bytes yourself (hex, base64) if a table genuinely needs them.
|
|
40
|
+
*/
|
|
41
|
+
import { type D1BindingLike } from './remote.js';
|
|
42
|
+
import { type D1LikeDatabase } from './types.js';
|
|
43
|
+
export interface HttpD1Options {
|
|
44
|
+
accountId: string;
|
|
45
|
+
databaseId: string;
|
|
46
|
+
/** An API token with D1 Edit on the account. */
|
|
47
|
+
apiToken: string;
|
|
48
|
+
/** Each request's ceiling, in ms. Default 30 s — a hung socket must not hang a job. */
|
|
49
|
+
timeoutMs?: number;
|
|
50
|
+
/** Injected for tests. Defaults to the global `fetch`. */
|
|
51
|
+
fetch?: typeof fetch;
|
|
52
|
+
/** Defaults to `https://api.cloudflare.com/client/v4`. */
|
|
53
|
+
apiBase?: string;
|
|
54
|
+
}
|
|
55
|
+
/** Thrown when the REST API itself refuses — auth, a missing database, a rate limit. */
|
|
56
|
+
export declare class D1HttpError extends Error {
|
|
57
|
+
readonly status: number;
|
|
58
|
+
constructor(status: number, message: string);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* The {@link D1BindingLike} half — exported for a caller that wants to wrap it itself
|
|
62
|
+
* (a test, or `perInvocation`). Most callers want {@link createHttpD1}.
|
|
63
|
+
*/
|
|
64
|
+
export declare function httpD1Binding(options: HttpD1Options): D1BindingLike;
|
|
65
|
+
/** A {@link D1LikeDatabase} over the D1 REST API. See the header for when, and when not. */
|
|
66
|
+
export declare function createHttpD1(options: HttpD1Options): D1LikeDatabase;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A D1 database reached over Cloudflare's REST API — for a process that is NOT a Worker.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this exists ─────────────────────────────────────────────────────────────────────
|
|
5
|
+
* An app that moves onto a Worker keeps some work on the Mac: a job that drives the owner's
|
|
6
|
+
* own Chrome, a bulk importer reading a local folder, a backfill, a backup. Before the move
|
|
7
|
+
* those wrote the Mac's `bun:sqlite` file; after it, that file is a frozen copy nothing reads,
|
|
8
|
+
* and a script writing into it reports success while the library it meant to change never
|
|
9
|
+
* hears about it. Measured on `music`'s cutover plan, 2026-09-22: four Mac-side writers, and
|
|
10
|
+
* the only other door into D1 was a per-app bearer route on the Worker for each one.
|
|
11
|
+
*
|
|
12
|
+
* So the Mac gets the same {@link D1LikeDatabase} a Worker holds, pointed at the same
|
|
13
|
+
* database, and the app's own catalogue code runs unchanged against production — ONE writer
|
|
14
|
+
* of the rows, one schema, one set of statements.
|
|
15
|
+
*
|
|
16
|
+
* ── How ──────────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
* It is a {@link D1BindingLike} over `POST /accounts/:account/d1/database/:db/query` (and
|
|
18
|
+
* `/raw`), handed to {@link createRemoteD1} — so every value normalisation, limit check and
|
|
19
|
+
* `success: false` guard is the binding driver's own, not a second copy that could drift.
|
|
20
|
+
* `batch()` sends `{ batch: [...] }`, which D1 runs as one unit exactly as a binding's batch
|
|
21
|
+
* does: measured 2026-09-23 on a throwaway database, a batch whose second statement violated
|
|
22
|
+
* a UNIQUE constraint left the first one's row absent.
|
|
23
|
+
*
|
|
24
|
+
* 🔴 **Not for a Worker.** A Worker has the binding, which is faster, free of an API token,
|
|
25
|
+
* and not subject to the REST API's rate limit. This is for a Mac process, with the account
|
|
26
|
+
* token read from the generation's secrets — never shipped to a browser or a Worker.
|
|
27
|
+
*
|
|
28
|
+
* 🔴 **Every number arrives as a REAL.** The REST API's parameters are JSON, and measured
|
|
29
|
+
* 2026-09-23 it binds a JSON `5` as the REAL `5.0` (`typeof(?)` answers `real`). Everywhere a
|
|
30
|
+
* number is compared, added, used as a `LIMIT`, or stored in an INTEGER column, that is
|
|
31
|
+
* indistinguishable from an integer — SQLite compares `5.0 = 5` true and INTEGER affinity
|
|
32
|
+
* stores `5`. The one place it differs: a number stored into a TEXT column is `'5.0'`, where
|
|
33
|
+
* the binding stores `'5'`. Bind a string into a TEXT column. (Sending integers as strings
|
|
34
|
+
* instead was measured too, and is worse: it breaks every comparison that has no column
|
|
35
|
+
* affinity, such as `CASE WHEN ? = 1`.)
|
|
36
|
+
*
|
|
37
|
+
* 🔴 **A BLOB parameter is refused.** The REST API's parameters are JSON; there is no
|
|
38
|
+
* lossless spelling of bytes in it that D1 documents, and guessing one would store the
|
|
39
|
+
* wrong thing silently. Encode bytes yourself (hex, base64) if a table genuinely needs them.
|
|
40
|
+
*/
|
|
41
|
+
import { createRemoteD1 } from './remote.js';
|
|
42
|
+
import { D1UnsupportedError } from './types.js';
|
|
43
|
+
/** Thrown when the REST API itself refuses — auth, a missing database, a rate limit. */
|
|
44
|
+
export class D1HttpError extends Error {
|
|
45
|
+
status;
|
|
46
|
+
constructor(status, message) {
|
|
47
|
+
super(`D1 over HTTP refused the request (${status}): ${message}`);
|
|
48
|
+
this.name = 'D1HttpError';
|
|
49
|
+
this.status = status;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
function toWire(values) {
|
|
53
|
+
return values.map((value, index) => {
|
|
54
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
|
|
55
|
+
throw new D1UnsupportedError(`a BLOB parameter (parameter ${index + 1}) over the D1 REST API`, 'encode the bytes as text (hex or base64) — the REST API has no documented lossless BLOB spelling');
|
|
56
|
+
}
|
|
57
|
+
return value;
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* The {@link D1BindingLike} half — exported for a caller that wants to wrap it itself
|
|
62
|
+
* (a test, or `perInvocation`). Most callers want {@link createHttpD1}.
|
|
63
|
+
*/
|
|
64
|
+
export function httpD1Binding(options) {
|
|
65
|
+
const doFetch = options.fetch ?? fetch;
|
|
66
|
+
const base = `${options.apiBase ?? 'https://api.cloudflare.com/client/v4'}/accounts/${options.accountId}/d1/database/${options.databaseId}`;
|
|
67
|
+
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
68
|
+
const post = async (path, body) => {
|
|
69
|
+
const response = await doFetch(`${base}${path}`, {
|
|
70
|
+
method: 'POST',
|
|
71
|
+
headers: { authorization: `Bearer ${options.apiToken}`, 'content-type': 'application/json' },
|
|
72
|
+
body: JSON.stringify(body),
|
|
73
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
74
|
+
});
|
|
75
|
+
const envelope = (await response.json().catch(() => null));
|
|
76
|
+
if (!response.ok || !envelope || envelope.success === false) {
|
|
77
|
+
const message = envelope?.errors
|
|
78
|
+
?.map((e) => e.message)
|
|
79
|
+
.filter(Boolean)
|
|
80
|
+
.join('; ') || response.statusText || 'no body';
|
|
81
|
+
throw new D1HttpError(response.status, message);
|
|
82
|
+
}
|
|
83
|
+
return envelope.result ?? [];
|
|
84
|
+
};
|
|
85
|
+
const asResult = (wire) => ({
|
|
86
|
+
results: Array.isArray(wire?.results) ? wire?.results : [],
|
|
87
|
+
success: wire?.success !== false,
|
|
88
|
+
...(wire?.meta ? { meta: wire.meta } : {}),
|
|
89
|
+
});
|
|
90
|
+
class HttpStatement {
|
|
91
|
+
sql;
|
|
92
|
+
params;
|
|
93
|
+
constructor(sql, params = []) {
|
|
94
|
+
this.sql = sql;
|
|
95
|
+
this.params = params;
|
|
96
|
+
}
|
|
97
|
+
bind(...values) {
|
|
98
|
+
return new HttpStatement(this.sql, toWire(values));
|
|
99
|
+
}
|
|
100
|
+
async all() {
|
|
101
|
+
return asResult((await post('/query', { sql: this.sql, params: this.params }))[0]);
|
|
102
|
+
}
|
|
103
|
+
run() {
|
|
104
|
+
return this.all();
|
|
105
|
+
}
|
|
106
|
+
async first(column) {
|
|
107
|
+
const row = ((await this.all()).results?.[0] ?? null);
|
|
108
|
+
if (row === null)
|
|
109
|
+
return null;
|
|
110
|
+
return column === undefined ? row : (row[column] ?? null);
|
|
111
|
+
}
|
|
112
|
+
async raw() {
|
|
113
|
+
const wire = (await post('/raw', { sql: this.sql, params: this.params }))[0];
|
|
114
|
+
const results = wire?.results;
|
|
115
|
+
return results?.rows ?? [];
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
prepare: (sql) => new HttpStatement(sql),
|
|
120
|
+
async batch(statements) {
|
|
121
|
+
const batch = statements.map((s) => {
|
|
122
|
+
if (!(s instanceof HttpStatement)) {
|
|
123
|
+
throw new TypeError('batch() accepts only statements from this same HTTP D1 handle.');
|
|
124
|
+
}
|
|
125
|
+
return { sql: s.sql, params: s.params };
|
|
126
|
+
});
|
|
127
|
+
if (batch.length === 0)
|
|
128
|
+
return [];
|
|
129
|
+
return (await post('/query', { batch })).map(asResult);
|
|
130
|
+
},
|
|
131
|
+
async exec(sql) {
|
|
132
|
+
const results = await post('/query', { sql });
|
|
133
|
+
return {
|
|
134
|
+
count: results.length,
|
|
135
|
+
duration: results.reduce((sum, r) => sum + (r.meta?.duration ?? 0), 0),
|
|
136
|
+
};
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/** A {@link D1LikeDatabase} over the D1 REST API. See the header for when, and when not. */
|
|
141
|
+
export function createHttpD1(options) {
|
|
142
|
+
return createRemoteD1(httpD1Binding(options));
|
|
143
|
+
}
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
export { type BackupPoint, createTimeTravelBackup, type DatabaseBackup, type TimeTravelOpts, } from './backup.js';
|
|
18
18
|
export { createPendingWrites, type InvocationD1, type PendingWrites, perInvocation, type WriteCollector } from './invocation.js';
|
|
19
|
+
export { createHttpD1, D1HttpError, type HttpD1Options } from './http.js';
|
|
19
20
|
export { assertBatchSize, assertWithinLimits, chunkForBind, inArray, LIMITS } from './limits.js';
|
|
20
21
|
export { createLocalD1, refuseInteractiveTransaction } from './local.js';
|
|
21
22
|
export { createRemoteD1, type D1BindingLike, type D1BindingMeta, type D1BindingResult, type D1BindingStatement, } from './remote.js';
|
package/dist/server/d1/index.js
CHANGED
|
@@ -34,6 +34,7 @@ export { createTimeTravelBackup, } from './backup.js';
|
|
|
34
34
|
// of them, since business tables stay on raw statements. Measured 2026-09-18 against the
|
|
35
35
|
// published 4.3.0 tarball. `barrelsReachNoOptionalPeer.spec.ts` is what keeps it out.
|
|
36
36
|
export { createPendingWrites, perInvocation } from './invocation.js';
|
|
37
|
+
export { createHttpD1, D1HttpError } from './http.js';
|
|
37
38
|
export { assertBatchSize, assertWithinLimits, chunkForBind, inArray, LIMITS } from './limits.js';
|
|
38
39
|
export { createLocalD1, refuseInteractiveTransaction } from './local.js';
|
|
39
40
|
export { createRemoteD1, } from './remote.js';
|
|
@@ -82,3 +82,4 @@ export declare function createLocalBackup(opts: LocalBackupOpts): DatabaseBackup
|
|
|
82
82
|
* a host that genuinely has both.
|
|
83
83
|
*/
|
|
84
84
|
export declare function backupFor(flavor: 'local' | 'd1', local: () => LocalBackupOpts, remote: () => TimeTravelOpts): DatabaseBackup;
|
|
85
|
+
export { type BackupSourceChoice, type BackupSourceInput, backupTables, chooseBackupSource, type PullD1Options, pullD1ToSqlite, servedRuntime, type Wrangler, } from './pullD1.js';
|
|
@@ -108,3 +108,7 @@ export function createLocalBackup(opts) {
|
|
|
108
108
|
export function backupFor(flavor, local, remote) {
|
|
109
109
|
return flavor === 'local' ? createLocalBackup(local()) : createTimeTravelBackup(remote());
|
|
110
110
|
}
|
|
111
|
+
// After a Worker cutover the live rows are D1's, and a Mac-hosted backup must pull them down
|
|
112
|
+
// rather than snapshot the frozen file — see `./pullD1.ts`. Here, not in the `./d1` barrel,
|
|
113
|
+
// for the reason this file's header gives: it reaches `bun:sqlite`.
|
|
114
|
+
export { backupTables, chooseBackupSource, pullD1ToSqlite, servedRuntime, } from './pullD1.js';
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* After a Worker cutover, the Mac's nightly backup must snapshot the LIVE rows — D1 — and not
|
|
3
|
+
* the Mac's frozen file. The three pieces of that, which `apps/collections` wrote first
|
|
4
|
+
* (2026-09-19/22) and `apps/music` needed next, so they live here once:
|
|
5
|
+
*
|
|
6
|
+
* · {@link chooseBackupSource} — which side holds the live rows, asked of the production
|
|
7
|
+
* hostname's `/healthz` rather than of a config flag, and REFUSING when it cannot tell;
|
|
8
|
+
* · {@link servedRuntime} — that probe;
|
|
9
|
+
* · {@link pullD1ToSqlite} — the live database pulled down into a real SQLite file, table by
|
|
10
|
+
* table ({@link backupTables}), so the app's existing whole-file snapshot path is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* ## 🔴 Why the production hostname and not `cursed.app.runtime`
|
|
13
|
+
*
|
|
14
|
+
* An app declares `runtime: "worker"` when it is PORTED, which can be months before a single
|
|
15
|
+
* row moves. Keying on the marker points the nightly backup at an empty D1 the day the port
|
|
16
|
+
* lands. `/healthz` publishes `runtime` for exactly this question — ask the hostname who
|
|
17
|
+
* answered. And an unreadable answer REFUSES: guessing the Mac snapshots a frozen file after
|
|
18
|
+
* the cutover (a GREEN backup of nothing current — the worst failure a backup has), guessing
|
|
19
|
+
* D1 snapshots a stale copy while the Mac is live.
|
|
20
|
+
*
|
|
21
|
+
* ## 🔴 Why table by table, and through `wrangler`
|
|
22
|
+
*
|
|
23
|
+
* `wrangler d1 export` REFUSES a whole database holding a virtual table — *"cannot export
|
|
24
|
+
* databases with Virtual Tables (fts5)"*, measured on collections' first post-cutover night —
|
|
25
|
+
* and a per-table export is allowed. `wrangler` rather than the REST API because the export
|
|
26
|
+
* is typed SQL: the REST API's JSON cannot tell the REAL `5.0` from the INTEGER `5`
|
|
27
|
+
* (`./http.ts` has the measurement), and a restore must put back what was there.
|
|
28
|
+
*/
|
|
29
|
+
export type BackupSourceChoice = {
|
|
30
|
+
kind: 'file' | 'd1' | 'refuse';
|
|
31
|
+
why: string;
|
|
32
|
+
};
|
|
33
|
+
export interface BackupSourceInput {
|
|
34
|
+
/** `--from-file`: the operator overriding all of this on purpose. */
|
|
35
|
+
fromFileFlag: boolean;
|
|
36
|
+
/** `package.json`'s `cursed.app.runtime`. NOT sufficient on its own — see the header. */
|
|
37
|
+
runtimeMarker: string | undefined;
|
|
38
|
+
/** What the PRODUCTION hostname's `/healthz` said its `runtime` was; `null` = no usable answer. */
|
|
39
|
+
servedRuntime: string | null;
|
|
40
|
+
}
|
|
41
|
+
/** Which side a backup must read. Pure, so every branch is testable without a network. */
|
|
42
|
+
export declare function chooseBackupSource(input: BackupSourceInput): BackupSourceChoice;
|
|
43
|
+
/**
|
|
44
|
+
* What `/healthz` says `runtime` is at `url`, or `null` if no usable answer came back — which
|
|
45
|
+
* means UNKNOWN and is never conflated with "not the Worker". Three attempts, because one blip
|
|
46
|
+
* must not cost a night's backup; a short timeout, because a hung fetch leaves launchd with a
|
|
47
|
+
* job that never exits.
|
|
48
|
+
*/
|
|
49
|
+
export declare function servedRuntime(url: string, { attempts, sleep }?: {
|
|
50
|
+
attempts?: number | undefined;
|
|
51
|
+
sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
52
|
+
}): Promise<string | null>;
|
|
53
|
+
/** One `wrangler` invocation, as {@link pullD1ToSqlite} needs it. Injected so a spec runs none. */
|
|
54
|
+
export type Wrangler = (args: string[]) => {
|
|
55
|
+
code: number;
|
|
56
|
+
stdout: string;
|
|
57
|
+
stderr: string;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* The tables a snapshot carries, from D1's own `sqlite_master` rows — every real table, and
|
|
61
|
+
* none of what cannot or must not be re-inserted: a virtual table and its shadows (`_config`,
|
|
62
|
+
* `_data`, `_docsize`, `_idx`, `_content` — an inverted index, not rows), `_cf_*`
|
|
63
|
+
* (Cloudflare's bookkeeping), and every `sqlite_*` table except `sqlite_sequence`, whose
|
|
64
|
+
* AUTOINCREMENT high-water marks belong in a restore.
|
|
65
|
+
*/
|
|
66
|
+
export declare function backupTables(rows: ReadonlyArray<{
|
|
67
|
+
name: string;
|
|
68
|
+
sql: string | null;
|
|
69
|
+
}>): string[];
|
|
70
|
+
export interface PullD1Options {
|
|
71
|
+
/** The SQLite file to create. */
|
|
72
|
+
destination: string;
|
|
73
|
+
/** The D1 database's NAME as `wrangler.jsonc` declares it at the top level. */
|
|
74
|
+
database: string;
|
|
75
|
+
/** The whole schema — normally the app's `db/schema.sql`. */
|
|
76
|
+
schemaSql: string;
|
|
77
|
+
/** A table that must be listed, or the pull is refused as the wrong database. */
|
|
78
|
+
requiredTable: string;
|
|
79
|
+
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
80
|
+
afterLoadSql?: string;
|
|
81
|
+
wrangler: Wrangler;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Pull the live D1 database into a real SQLite file.
|
|
85
|
+
*
|
|
86
|
+
* 🔴 It THROWS rather than returning an empty database. Every layer beneath treats a readable
|
|
87
|
+
* SQLite file as success, so an export that produced nothing would sail through the
|
|
88
|
+
* magic-byte check and `integrity_check` and land on the shelf with today's date on it.
|
|
89
|
+
*/
|
|
90
|
+
export declare function pullD1ToSqlite(options: PullD1Options): Promise<void>;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* After a Worker cutover, the Mac's nightly backup must snapshot the LIVE rows — D1 — and not
|
|
3
|
+
* the Mac's frozen file. The three pieces of that, which `apps/collections` wrote first
|
|
4
|
+
* (2026-09-19/22) and `apps/music` needed next, so they live here once:
|
|
5
|
+
*
|
|
6
|
+
* · {@link chooseBackupSource} — which side holds the live rows, asked of the production
|
|
7
|
+
* hostname's `/healthz` rather than of a config flag, and REFUSING when it cannot tell;
|
|
8
|
+
* · {@link servedRuntime} — that probe;
|
|
9
|
+
* · {@link pullD1ToSqlite} — the live database pulled down into a real SQLite file, table by
|
|
10
|
+
* table ({@link backupTables}), so the app's existing whole-file snapshot path is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* ## 🔴 Why the production hostname and not `cursed.app.runtime`
|
|
13
|
+
*
|
|
14
|
+
* An app declares `runtime: "worker"` when it is PORTED, which can be months before a single
|
|
15
|
+
* row moves. Keying on the marker points the nightly backup at an empty D1 the day the port
|
|
16
|
+
* lands. `/healthz` publishes `runtime` for exactly this question — ask the hostname who
|
|
17
|
+
* answered. And an unreadable answer REFUSES: guessing the Mac snapshots a frozen file after
|
|
18
|
+
* the cutover (a GREEN backup of nothing current — the worst failure a backup has), guessing
|
|
19
|
+
* D1 snapshots a stale copy while the Mac is live.
|
|
20
|
+
*
|
|
21
|
+
* ## 🔴 Why table by table, and through `wrangler`
|
|
22
|
+
*
|
|
23
|
+
* `wrangler d1 export` REFUSES a whole database holding a virtual table — *"cannot export
|
|
24
|
+
* databases with Virtual Tables (fts5)"*, measured on collections' first post-cutover night —
|
|
25
|
+
* and a per-table export is allowed. `wrangler` rather than the REST API because the export
|
|
26
|
+
* is typed SQL: the REST API's JSON cannot tell the REAL `5.0` from the INTEGER `5`
|
|
27
|
+
* (`./http.ts` has the measurement), and a restore must put back what was there.
|
|
28
|
+
*/
|
|
29
|
+
import { Database } from 'bun:sqlite';
|
|
30
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
31
|
+
/** Which side a backup must read. Pure, so every branch is testable without a network. */
|
|
32
|
+
export function chooseBackupSource(input) {
|
|
33
|
+
if (input.fromFileFlag) {
|
|
34
|
+
return { kind: 'file', why: "--from-file was given: snapshotting this Mac's sqlite on purpose" };
|
|
35
|
+
}
|
|
36
|
+
if (input.runtimeMarker !== 'worker') {
|
|
37
|
+
return { kind: 'file', why: 'this app does not declare a worker runtime, so the Mac holds the live rows' };
|
|
38
|
+
}
|
|
39
|
+
if (input.servedRuntime === null) {
|
|
40
|
+
return {
|
|
41
|
+
kind: 'refuse',
|
|
42
|
+
why: "could not read `runtime` from the production hostname's /healthz, so which side holds the live " +
|
|
43
|
+
'rows is unknown. Guessing the Mac would snapshot a database that may have been frozen at the ' +
|
|
44
|
+
'cutover; guessing D1 would snapshot a stale copy while the Mac is still serving. Neither is a ' +
|
|
45
|
+
'backup. Fix the probe, or run `bun run backup -- --from-file` if you know the Mac is live.',
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
if (input.servedRuntime === 'worker') {
|
|
49
|
+
return { kind: 'd1', why: 'the production hostname is served by the Worker, so the live rows are in D1' };
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
kind: 'file',
|
|
53
|
+
why: `the production hostname is still served by '${input.servedRuntime}', so the Mac holds the live rows`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* What `/healthz` says `runtime` is at `url`, or `null` if no usable answer came back — which
|
|
58
|
+
* means UNKNOWN and is never conflated with "not the Worker". Three attempts, because one blip
|
|
59
|
+
* must not cost a night's backup; a short timeout, because a hung fetch leaves launchd with a
|
|
60
|
+
* job that never exits.
|
|
61
|
+
*/
|
|
62
|
+
export async function servedRuntime(url, { attempts = 3, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
|
|
63
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
64
|
+
try {
|
|
65
|
+
const response = await fetch(`${url.replace(/\/+$/, '')}/healthz`, {
|
|
66
|
+
signal: AbortSignal.timeout(10_000),
|
|
67
|
+
headers: { 'cache-control': 'no-cache' },
|
|
68
|
+
});
|
|
69
|
+
if (!response.ok)
|
|
70
|
+
throw new Error(`HTTP ${response.status}`);
|
|
71
|
+
const body = (await response.json());
|
|
72
|
+
// A body with no `runtime` is not an answer.
|
|
73
|
+
if (typeof body.runtime === 'string' && body.runtime)
|
|
74
|
+
return body.runtime;
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
if (attempt === attempts)
|
|
79
|
+
return null;
|
|
80
|
+
await sleep(2000 * attempt);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The tables a snapshot carries, from D1's own `sqlite_master` rows — every real table, and
|
|
87
|
+
* none of what cannot or must not be re-inserted: a virtual table and its shadows (`_config`,
|
|
88
|
+
* `_data`, `_docsize`, `_idx`, `_content` — an inverted index, not rows), `_cf_*`
|
|
89
|
+
* (Cloudflare's bookkeeping), and every `sqlite_*` table except `sqlite_sequence`, whose
|
|
90
|
+
* AUTOINCREMENT high-water marks belong in a restore.
|
|
91
|
+
*/
|
|
92
|
+
export function backupTables(rows) {
|
|
93
|
+
const virtual = rows.filter((r) => /^\s*CREATE\s+VIRTUAL\s+TABLE/i.test(r.sql ?? '')).map((r) => r.name);
|
|
94
|
+
const shadow = new Set(virtual.flatMap((vt) => ['config', 'data', 'docsize', 'idx', 'content'].map((suffix) => `${vt}_${suffix}`)));
|
|
95
|
+
return rows
|
|
96
|
+
.map((r) => r.name)
|
|
97
|
+
.filter((name) => !virtual.includes(name) && !shadow.has(name))
|
|
98
|
+
.filter((name) => !name.startsWith('_cf_'))
|
|
99
|
+
.filter((name) => name === 'sqlite_sequence' || !name.startsWith('sqlite_'))
|
|
100
|
+
.sort();
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Pull the live D1 database into a real SQLite file.
|
|
104
|
+
*
|
|
105
|
+
* 🔴 It THROWS rather than returning an empty database. Every layer beneath treats a readable
|
|
106
|
+
* SQLite file as success, so an export that produced nothing would sail through the
|
|
107
|
+
* magic-byte check and `integrity_check` and land on the shelf with today's date on it.
|
|
108
|
+
*/
|
|
109
|
+
export async function pullD1ToSqlite(options) {
|
|
110
|
+
const { destination, database, wrangler } = options;
|
|
111
|
+
const listed = wrangler([
|
|
112
|
+
'd1', 'execute', database, '--remote', '--env=', '--json',
|
|
113
|
+
'--command', "select name, sql from sqlite_master where type = 'table'",
|
|
114
|
+
]);
|
|
115
|
+
if (listed.code !== 0)
|
|
116
|
+
throw new Error(`could not list D1's tables: ${listed.stderr.trim() || listed.stdout.trim()}`);
|
|
117
|
+
const json = listed.stdout.slice(listed.stdout.indexOf('['));
|
|
118
|
+
const rows = JSON.parse(json)[0]?.results ?? [];
|
|
119
|
+
const tables = backupTables(rows);
|
|
120
|
+
if (!tables.includes(options.requiredTable)) {
|
|
121
|
+
throw new Error(`D1 lists no \`${options.requiredTable}\` table (saw: ${rows.map((r) => r.name).join(', ') || 'nothing'})`);
|
|
122
|
+
}
|
|
123
|
+
let sql = '';
|
|
124
|
+
for (const table of tables) {
|
|
125
|
+
const dump = `${destination}.${table}.sql`;
|
|
126
|
+
const exported = wrangler(['d1', 'export', database, '--remote', '--env=', '--table', table, '--no-schema', '--output', dump, '-y']);
|
|
127
|
+
if (exported.code !== 0) {
|
|
128
|
+
throw new Error(`wrangler d1 export --table ${table} failed: ${exported.stderr.trim() || exported.stdout.trim()}`);
|
|
129
|
+
}
|
|
130
|
+
if (!existsSync(dump))
|
|
131
|
+
throw new Error(`wrangler d1 export --table ${table} reported success and wrote no file at ${dump}`);
|
|
132
|
+
sql += `${readFileSync(dump, 'utf8')}\n`;
|
|
133
|
+
}
|
|
134
|
+
if (!/INSERT INTO/i.test(sql)) {
|
|
135
|
+
throw new Error('the D1 export contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
136
|
+
}
|
|
137
|
+
const db = new Database(destination, { create: true });
|
|
138
|
+
try {
|
|
139
|
+
// `exec`, not `run`: each is many statements and `run` would execute only the first,
|
|
140
|
+
// leaving a file with a schema and no rows that looks perfectly valid.
|
|
141
|
+
db.exec(options.schemaSql);
|
|
142
|
+
db.exec(sql);
|
|
143
|
+
if (options.afterLoadSql)
|
|
144
|
+
db.exec(options.afterLoadSql);
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
db.close();
|
|
148
|
+
}
|
|
149
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.17.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { createHttpD1, D1HttpError } from './http.js';
|
|
3
|
+
import { D1UnsupportedError } from './types.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The HTTP driver against a fake of the D1 REST API's WIRE shapes — the envelopes below are
|
|
7
|
+
* copied from real responses measured 2026-09-23 on a throwaway database (see `http.ts`).
|
|
8
|
+
*/
|
|
9
|
+
interface Sent {
|
|
10
|
+
url: string;
|
|
11
|
+
body: Record<string, unknown>;
|
|
12
|
+
auth: string | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function fakeApi(reply: (path: string, body: Record<string, unknown>) => { status?: number; envelope: unknown }) {
|
|
16
|
+
const sent: Sent[] = [];
|
|
17
|
+
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
18
|
+
const url = String(input);
|
|
19
|
+
const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
|
|
20
|
+
sent.push({ url, body, auth: new Headers(init?.headers).get('authorization') });
|
|
21
|
+
const path = url.endsWith('/raw') ? '/raw' : '/query';
|
|
22
|
+
const { status = 200, envelope } = reply(path, body);
|
|
23
|
+
return new Response(JSON.stringify(envelope), { status });
|
|
24
|
+
}) as typeof fetch;
|
|
25
|
+
const db = createHttpD1({ accountId: 'acct', databaseId: 'db1', apiToken: 'tok', fetch: fetchImpl });
|
|
26
|
+
return { db, sent };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const ok = (result: unknown[]) => ({ envelope: { success: true, errors: [], result } });
|
|
30
|
+
const meta = { duration: 0.4, changes: 1, last_row_id: 7, rows_read: 1, rows_written: 1 };
|
|
31
|
+
|
|
32
|
+
describe('createHttpD1', () => {
|
|
33
|
+
test('a statement is one POST to /query with its sql, params and the bearer', async () => {
|
|
34
|
+
const { db, sent } = fakeApi(() => ok([{ results: [{ id: 1, v: 'a' }], success: true, meta }]));
|
|
35
|
+
const rows = await db.prepare('SELECT * FROM t WHERE id = ? AND v = ?').bind(1, 'a').all();
|
|
36
|
+
expect(rows.results).toEqual([{ id: 1, v: 'a' }]);
|
|
37
|
+
expect(sent).toEqual([
|
|
38
|
+
{
|
|
39
|
+
url: 'https://api.cloudflare.com/client/v4/accounts/acct/d1/database/db1/query',
|
|
40
|
+
body: { sql: 'SELECT * FROM t WHERE id = ? AND v = ?', params: [1, 'a'] },
|
|
41
|
+
auth: 'Bearer tok',
|
|
42
|
+
},
|
|
43
|
+
]);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('run reports changes and last_row_id; first reads a row and a column', async () => {
|
|
47
|
+
const { db } = fakeApi(() => ok([{ results: [{ n: 3 }], success: true, meta }]));
|
|
48
|
+
const res = await db.prepare('INSERT INTO t (v) VALUES (?)').bind('x').run();
|
|
49
|
+
expect(res.meta.changes).toBe(1);
|
|
50
|
+
expect(res.meta.last_row_id).toBe(7);
|
|
51
|
+
expect(await db.prepare('SELECT COUNT(*) n FROM t').first()).toEqual({ n: 3 });
|
|
52
|
+
expect(await db.prepare('SELECT COUNT(*) n FROM t').first('n')).toBe(3);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('batch is ONE request carrying every statement, and answers one result each', async () => {
|
|
56
|
+
const { db, sent } = fakeApi(() =>
|
|
57
|
+
ok([
|
|
58
|
+
{ results: [], success: true, meta },
|
|
59
|
+
{ results: [{ id: 1 }], success: true, meta: { ...meta, changes: 0 } },
|
|
60
|
+
]),
|
|
61
|
+
);
|
|
62
|
+
const results = await db.batch([
|
|
63
|
+
db.prepare('INSERT INTO t (v) VALUES (?)').bind('a'),
|
|
64
|
+
db.prepare('SELECT id FROM t'),
|
|
65
|
+
]);
|
|
66
|
+
expect(sent).toHaveLength(1);
|
|
67
|
+
expect(sent[0]?.body).toEqual({
|
|
68
|
+
batch: [
|
|
69
|
+
{ sql: 'INSERT INTO t (v) VALUES (?)', params: ['a'] },
|
|
70
|
+
{ sql: 'SELECT id FROM t', params: [] },
|
|
71
|
+
],
|
|
72
|
+
});
|
|
73
|
+
expect(results.map((r) => r.results)).toEqual([[], [{ id: 1 }]]);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('raw reads the /raw endpoint, so duplicate column names survive', async () => {
|
|
77
|
+
const { db, sent } = fakeApi(() =>
|
|
78
|
+
ok([{ results: { columns: ['id', 'v', 'v'], rows: [[1, 'a', 'a']] }, success: true, meta }]),
|
|
79
|
+
);
|
|
80
|
+
expect(await db.prepare('SELECT id, v, v AS v FROM t').raw()).toEqual([[1, 'a', 'a']]);
|
|
81
|
+
expect(sent[0]?.url.endsWith('/raw')).toBe(true);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test('a refused statement throws with D1 own message — never a silent empty result', async () => {
|
|
85
|
+
const { db } = fakeApi(() => ({
|
|
86
|
+
status: 400,
|
|
87
|
+
envelope: {
|
|
88
|
+
success: false,
|
|
89
|
+
result: [],
|
|
90
|
+
errors: [{ code: 7500, message: 'UNIQUE constraint failed: t.v: SQLITE_CONSTRAINT' }],
|
|
91
|
+
},
|
|
92
|
+
}));
|
|
93
|
+
const attempt = db.prepare('INSERT INTO t (v) VALUES (?)').bind('one').run();
|
|
94
|
+
await expect(attempt).rejects.toBeInstanceOf(D1HttpError);
|
|
95
|
+
await expect(db.prepare('INSERT INTO t (v) VALUES (?)').bind('one').run()).rejects.toThrow(
|
|
96
|
+
/400\): UNIQUE constraint failed/,
|
|
97
|
+
);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('a body that is not JSON (a proxy page, an outage) still throws with the status', async () => {
|
|
101
|
+
const fetchImpl = (async () => new Response('<html>bad gateway</html>', { status: 502 })) as unknown as typeof fetch;
|
|
102
|
+
const db = createHttpD1({ accountId: 'a', databaseId: 'd', apiToken: 't', fetch: fetchImpl });
|
|
103
|
+
await expect(db.prepare('SELECT 1').all()).rejects.toThrow(/\(502\)/);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('a BLOB parameter is refused rather than sent in a spelling D1 does not document', () => {
|
|
107
|
+
const { db } = fakeApi(() => ok([]));
|
|
108
|
+
expect(() => db.prepare('INSERT INTO t (b) VALUES (?)').bind(new Uint8Array([1, 2]))).toThrow(D1UnsupportedError);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test('the parameter cap is the binding driver own — asserted before anything is sent', () => {
|
|
112
|
+
const { db, sent } = fakeApi(() => ok([]));
|
|
113
|
+
const holes = Array.from({ length: 101 }, () => '?').join(', ');
|
|
114
|
+
expect(() => db.prepare(`SELECT ${holes}`).bind(...Array.from({ length: 101 }, (_, i) => i))).toThrow();
|
|
115
|
+
expect(sent).toHaveLength(0);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test('exec sends the whole script and counts the statements D1 ran', async () => {
|
|
119
|
+
const { db, sent } = fakeApi(() => ok([{ meta }, { meta }, { meta }]));
|
|
120
|
+
expect(await db.exec('CREATE TABLE u (x); INSERT INTO u VALUES (1); INSERT INTO u VALUES (2)')).toEqual({
|
|
121
|
+
count: 3,
|
|
122
|
+
duration: 1.2000000000000002,
|
|
123
|
+
});
|
|
124
|
+
expect(sent[0]?.body).toEqual({ sql: 'CREATE TABLE u (x); INSERT INTO u VALUES (1); INSERT INTO u VALUES (2)' });
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A D1 database reached over Cloudflare's REST API — for a process that is NOT a Worker.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this exists ─────────────────────────────────────────────────────────────────────
|
|
5
|
+
* An app that moves onto a Worker keeps some work on the Mac: a job that drives the owner's
|
|
6
|
+
* own Chrome, a bulk importer reading a local folder, a backfill, a backup. Before the move
|
|
7
|
+
* those wrote the Mac's `bun:sqlite` file; after it, that file is a frozen copy nothing reads,
|
|
8
|
+
* and a script writing into it reports success while the library it meant to change never
|
|
9
|
+
* hears about it. Measured on `music`'s cutover plan, 2026-09-22: four Mac-side writers, and
|
|
10
|
+
* the only other door into D1 was a per-app bearer route on the Worker for each one.
|
|
11
|
+
*
|
|
12
|
+
* So the Mac gets the same {@link D1LikeDatabase} a Worker holds, pointed at the same
|
|
13
|
+
* database, and the app's own catalogue code runs unchanged against production — ONE writer
|
|
14
|
+
* of the rows, one schema, one set of statements.
|
|
15
|
+
*
|
|
16
|
+
* ── How ──────────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
* It is a {@link D1BindingLike} over `POST /accounts/:account/d1/database/:db/query` (and
|
|
18
|
+
* `/raw`), handed to {@link createRemoteD1} — so every value normalisation, limit check and
|
|
19
|
+
* `success: false` guard is the binding driver's own, not a second copy that could drift.
|
|
20
|
+
* `batch()` sends `{ batch: [...] }`, which D1 runs as one unit exactly as a binding's batch
|
|
21
|
+
* does: measured 2026-09-23 on a throwaway database, a batch whose second statement violated
|
|
22
|
+
* a UNIQUE constraint left the first one's row absent.
|
|
23
|
+
*
|
|
24
|
+
* 🔴 **Not for a Worker.** A Worker has the binding, which is faster, free of an API token,
|
|
25
|
+
* and not subject to the REST API's rate limit. This is for a Mac process, with the account
|
|
26
|
+
* token read from the generation's secrets — never shipped to a browser or a Worker.
|
|
27
|
+
*
|
|
28
|
+
* 🔴 **Every number arrives as a REAL.** The REST API's parameters are JSON, and measured
|
|
29
|
+
* 2026-09-23 it binds a JSON `5` as the REAL `5.0` (`typeof(?)` answers `real`). Everywhere a
|
|
30
|
+
* number is compared, added, used as a `LIMIT`, or stored in an INTEGER column, that is
|
|
31
|
+
* indistinguishable from an integer — SQLite compares `5.0 = 5` true and INTEGER affinity
|
|
32
|
+
* stores `5`. The one place it differs: a number stored into a TEXT column is `'5.0'`, where
|
|
33
|
+
* the binding stores `'5'`. Bind a string into a TEXT column. (Sending integers as strings
|
|
34
|
+
* instead was measured too, and is worse: it breaks every comparison that has no column
|
|
35
|
+
* affinity, such as `CASE WHEN ? = 1`.)
|
|
36
|
+
*
|
|
37
|
+
* 🔴 **A BLOB parameter is refused.** The REST API's parameters are JSON; there is no
|
|
38
|
+
* lossless spelling of bytes in it that D1 documents, and guessing one would store the
|
|
39
|
+
* wrong thing silently. Encode bytes yourself (hex, base64) if a table genuinely needs them.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { createRemoteD1, type D1BindingLike, type D1BindingResult, type D1BindingStatement } from './remote.js';
|
|
43
|
+
import { D1UnsupportedError, type D1LikeDatabase } from './types.js';
|
|
44
|
+
|
|
45
|
+
export interface HttpD1Options {
|
|
46
|
+
accountId: string;
|
|
47
|
+
databaseId: string;
|
|
48
|
+
/** An API token with D1 Edit on the account. */
|
|
49
|
+
apiToken: string;
|
|
50
|
+
/** Each request's ceiling, in ms. Default 30 s — a hung socket must not hang a job. */
|
|
51
|
+
timeoutMs?: number;
|
|
52
|
+
/** Injected for tests. Defaults to the global `fetch`. */
|
|
53
|
+
fetch?: typeof fetch;
|
|
54
|
+
/** Defaults to `https://api.cloudflare.com/client/v4`. */
|
|
55
|
+
apiBase?: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
interface WireResult {
|
|
59
|
+
results?: unknown;
|
|
60
|
+
success?: boolean;
|
|
61
|
+
meta?: D1BindingResult['meta'];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
interface Envelope {
|
|
65
|
+
success?: boolean;
|
|
66
|
+
errors?: { code?: number; message?: string }[];
|
|
67
|
+
result?: WireResult[];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Thrown when the REST API itself refuses — auth, a missing database, a rate limit. */
|
|
71
|
+
export class D1HttpError extends Error {
|
|
72
|
+
readonly status: number;
|
|
73
|
+
constructor(status: number, message: string) {
|
|
74
|
+
super(`D1 over HTTP refused the request (${status}): ${message}`);
|
|
75
|
+
this.name = 'D1HttpError';
|
|
76
|
+
this.status = status;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function toWire(values: readonly unknown[]): unknown[] {
|
|
81
|
+
return values.map((value, index) => {
|
|
82
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
|
|
83
|
+
throw new D1UnsupportedError(
|
|
84
|
+
`a BLOB parameter (parameter ${index + 1}) over the D1 REST API`,
|
|
85
|
+
'encode the bytes as text (hex or base64) — the REST API has no documented lossless BLOB spelling',
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
return value;
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The {@link D1BindingLike} half — exported for a caller that wants to wrap it itself
|
|
94
|
+
* (a test, or `perInvocation`). Most callers want {@link createHttpD1}.
|
|
95
|
+
*/
|
|
96
|
+
export function httpD1Binding(options: HttpD1Options): D1BindingLike {
|
|
97
|
+
const doFetch = options.fetch ?? fetch;
|
|
98
|
+
const base = `${options.apiBase ?? 'https://api.cloudflare.com/client/v4'}/accounts/${options.accountId}/d1/database/${options.databaseId}`;
|
|
99
|
+
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
100
|
+
|
|
101
|
+
const post = async (path: '/query' | '/raw', body: unknown): Promise<WireResult[]> => {
|
|
102
|
+
const response = await doFetch(`${base}${path}`, {
|
|
103
|
+
method: 'POST',
|
|
104
|
+
headers: { authorization: `Bearer ${options.apiToken}`, 'content-type': 'application/json' },
|
|
105
|
+
body: JSON.stringify(body),
|
|
106
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
107
|
+
});
|
|
108
|
+
const envelope = (await response.json().catch(() => null)) as Envelope | null;
|
|
109
|
+
if (!response.ok || !envelope || envelope.success === false) {
|
|
110
|
+
const message =
|
|
111
|
+
envelope?.errors
|
|
112
|
+
?.map((e) => e.message)
|
|
113
|
+
.filter(Boolean)
|
|
114
|
+
.join('; ') || response.statusText || 'no body';
|
|
115
|
+
throw new D1HttpError(response.status, message);
|
|
116
|
+
}
|
|
117
|
+
return envelope.result ?? [];
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const asResult = (wire: WireResult | undefined): D1BindingResult => ({
|
|
121
|
+
results: Array.isArray(wire?.results) ? (wire?.results as unknown[]) : [],
|
|
122
|
+
success: wire?.success !== false,
|
|
123
|
+
...(wire?.meta ? { meta: wire.meta } : {}),
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
class HttpStatement implements D1BindingStatement {
|
|
127
|
+
constructor(
|
|
128
|
+
readonly sql: string,
|
|
129
|
+
readonly params: unknown[] = [],
|
|
130
|
+
) {}
|
|
131
|
+
|
|
132
|
+
bind(...values: unknown[]): D1BindingStatement {
|
|
133
|
+
return new HttpStatement(this.sql, toWire(values));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async all(): Promise<D1BindingResult> {
|
|
137
|
+
return asResult((await post('/query', { sql: this.sql, params: this.params }))[0]);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
run(): Promise<D1BindingResult> {
|
|
141
|
+
return this.all();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async first(column?: string): Promise<unknown> {
|
|
145
|
+
const row = ((await this.all()).results?.[0] ?? null) as Record<string, unknown> | null;
|
|
146
|
+
if (row === null) return null;
|
|
147
|
+
return column === undefined ? row : (row[column] ?? null);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
async raw(): Promise<unknown[][]> {
|
|
151
|
+
const wire = (await post('/raw', { sql: this.sql, params: this.params }))[0];
|
|
152
|
+
const results = wire?.results as { rows?: unknown[][] } | undefined;
|
|
153
|
+
return results?.rows ?? [];
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
prepare: (sql) => new HttpStatement(sql),
|
|
159
|
+
async batch(statements) {
|
|
160
|
+
const batch = statements.map((s) => {
|
|
161
|
+
if (!(s instanceof HttpStatement)) {
|
|
162
|
+
throw new TypeError('batch() accepts only statements from this same HTTP D1 handle.');
|
|
163
|
+
}
|
|
164
|
+
return { sql: s.sql, params: s.params };
|
|
165
|
+
});
|
|
166
|
+
if (batch.length === 0) return [];
|
|
167
|
+
return (await post('/query', { batch })).map(asResult);
|
|
168
|
+
},
|
|
169
|
+
async exec(sql) {
|
|
170
|
+
const results = await post('/query', { sql });
|
|
171
|
+
return {
|
|
172
|
+
count: results.length,
|
|
173
|
+
duration: results.reduce((sum, r) => sum + (r.meta?.duration ?? 0), 0),
|
|
174
|
+
};
|
|
175
|
+
},
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** A {@link D1LikeDatabase} over the D1 REST API. See the header for when, and when not. */
|
|
180
|
+
export function createHttpD1(options: HttpD1Options): D1LikeDatabase {
|
|
181
|
+
return createRemoteD1(httpD1Binding(options));
|
|
182
|
+
}
|
package/src/server/d1/index.ts
CHANGED
|
@@ -40,6 +40,7 @@ export {
|
|
|
40
40
|
// of them, since business tables stay on raw statements. Measured 2026-09-18 against the
|
|
41
41
|
// published 4.3.0 tarball. `barrelsReachNoOptionalPeer.spec.ts` is what keeps it out.
|
|
42
42
|
export { createPendingWrites, type InvocationD1, type PendingWrites, perInvocation, type WriteCollector } from './invocation.js';
|
|
43
|
+
export { createHttpD1, D1HttpError, type HttpD1Options } from './http.js';
|
|
43
44
|
export { assertBatchSize, assertWithinLimits, chunkForBind, inArray, LIMITS } from './limits.js';
|
|
44
45
|
export { createLocalD1, refuseInteractiveTransaction } from './local.js';
|
|
45
46
|
export {
|
|
@@ -135,3 +135,17 @@ export function backupFor(
|
|
|
135
135
|
): DatabaseBackup {
|
|
136
136
|
return flavor === 'local' ? createLocalBackup(local()) : createTimeTravelBackup(remote());
|
|
137
137
|
}
|
|
138
|
+
|
|
139
|
+
// After a Worker cutover the live rows are D1's, and a Mac-hosted backup must pull them down
|
|
140
|
+
// rather than snapshot the frozen file — see `./pullD1.ts`. Here, not in the `./d1` barrel,
|
|
141
|
+
// for the reason this file's header gives: it reaches `bun:sqlite`.
|
|
142
|
+
export {
|
|
143
|
+
type BackupSourceChoice,
|
|
144
|
+
type BackupSourceInput,
|
|
145
|
+
backupTables,
|
|
146
|
+
chooseBackupSource,
|
|
147
|
+
type PullD1Options,
|
|
148
|
+
pullD1ToSqlite,
|
|
149
|
+
servedRuntime,
|
|
150
|
+
type Wrangler,
|
|
151
|
+
} from './pullD1.js';
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { Database } from 'bun:sqlite';
|
|
2
|
+
import { afterEach, describe, expect, test } from 'bun:test';
|
|
3
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { backupTables, chooseBackupSource, pullD1ToSqlite, servedRuntime, type Wrangler } from './localBackup.js';
|
|
7
|
+
|
|
8
|
+
const dirs: string[] = [];
|
|
9
|
+
const scratch = (): string => {
|
|
10
|
+
const dir = mkdtempSync(join(tmpdir(), 'd1-pull-'));
|
|
11
|
+
dirs.push(dir);
|
|
12
|
+
return dir;
|
|
13
|
+
};
|
|
14
|
+
afterEach(() => {
|
|
15
|
+
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true });
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
describe('🔴 chooseBackupSource — the one whose failure looks like a green backup', () => {
|
|
19
|
+
const WORKER = 'worker';
|
|
20
|
+
|
|
21
|
+
test('a ported app whose production hostname is still the Mac reads the MAC file', () => {
|
|
22
|
+
const choice = chooseBackupSource({ fromFileFlag: false, runtimeMarker: WORKER, servedRuntime: 'bun' });
|
|
23
|
+
expect(choice.kind).toBe('file');
|
|
24
|
+
expect(choice.why).toContain("still served by 'bun'");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('once the production hostname answers as the Worker, it pulls D1', () => {
|
|
28
|
+
expect(chooseBackupSource({ fromFileFlag: false, runtimeMarker: WORKER, servedRuntime: WORKER }).kind).toBe('d1');
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('🔴 an unreadable probe REFUSES — it never guesses a side', () => {
|
|
32
|
+
const choice = chooseBackupSource({ fromFileFlag: false, runtimeMarker: WORKER, servedRuntime: null });
|
|
33
|
+
expect(choice.kind).toBe('refuse');
|
|
34
|
+
expect(choice.why).toContain('--from-file');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('an app that never declared a worker runtime never pulls', () => {
|
|
38
|
+
for (const marker of [undefined, 'bun', '']) {
|
|
39
|
+
expect(chooseBackupSource({ fromFileFlag: false, runtimeMarker: marker, servedRuntime: null }).kind).toBe('file');
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('--from-file wins over everything, including a live Worker', () => {
|
|
44
|
+
const choice = chooseBackupSource({ fromFileFlag: true, runtimeMarker: WORKER, servedRuntime: WORKER });
|
|
45
|
+
expect(choice.kind).toBe('file');
|
|
46
|
+
expect(choice.why).toContain('on purpose');
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe('servedRuntime', () => {
|
|
51
|
+
const serve = (body: unknown, status = 200) =>
|
|
52
|
+
Bun.serve({ port: 0, fetch: () => Response.json(body, { status }) });
|
|
53
|
+
|
|
54
|
+
test('reads `runtime` off /healthz', async () => {
|
|
55
|
+
const server = serve({ runtime: 'worker' });
|
|
56
|
+
try {
|
|
57
|
+
expect(await servedRuntime(`http://127.0.0.1:${server.port}`)).toBe('worker');
|
|
58
|
+
} finally {
|
|
59
|
+
server.stop(true);
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('🔴 a body with no runtime, or an error, is UNKNOWN — never "not the Worker"', async () => {
|
|
64
|
+
const bare = serve({ ok: true });
|
|
65
|
+
const broken = serve({ runtime: 'worker' }, 503);
|
|
66
|
+
try {
|
|
67
|
+
const noSleep = { attempts: 2, sleep: async () => undefined };
|
|
68
|
+
expect(await servedRuntime(`http://127.0.0.1:${bare.port}`, noSleep)).toBeNull();
|
|
69
|
+
expect(await servedRuntime(`http://127.0.0.1:${broken.port}`, noSleep)).toBeNull();
|
|
70
|
+
} finally {
|
|
71
|
+
bare.stop(true);
|
|
72
|
+
broken.stop(true);
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
describe('pulling D1 down — table by table, because a whole-database export refuses fts5', () => {
|
|
78
|
+
const MASTER = [
|
|
79
|
+
{ name: '_cf_KV', sql: 'CREATE TABLE _cf_KV (key TEXT PRIMARY KEY)' },
|
|
80
|
+
{ name: 'items', sql: 'CREATE TABLE items ( id TEXT PRIMARY KEY )' },
|
|
81
|
+
{ name: 'items_fts', sql: 'CREATE VIRTUAL TABLE items_fts USING fts5(text)' },
|
|
82
|
+
{ name: 'items_fts_config', sql: "CREATE TABLE 'items_fts_config'(k PRIMARY KEY, v)" },
|
|
83
|
+
{ name: 'items_fts_data', sql: "CREATE TABLE 'items_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" },
|
|
84
|
+
{ name: 'items_fts_docsize', sql: "CREATE TABLE 'items_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" },
|
|
85
|
+
{ name: 'items_fts_idx', sql: "CREATE TABLE 'items_fts_idx'(segid, term, pgno)" },
|
|
86
|
+
{ name: 'items_fts_map', sql: 'CREATE TABLE items_fts_map ( rowid INTEGER PRIMARY KEY AUTOINCREMENT )' },
|
|
87
|
+
{ name: 'meta', sql: 'CREATE TABLE meta ( key TEXT PRIMARY KEY )' },
|
|
88
|
+
{ name: 'sqlite_sequence', sql: 'CREATE TABLE sqlite_sequence(name,seq)' },
|
|
89
|
+
{ name: 'sqlite_stat1', sql: 'CREATE TABLE sqlite_stat1(tbl,idx,stat)' },
|
|
90
|
+
];
|
|
91
|
+
const SCHEMA = [
|
|
92
|
+
'CREATE TABLE items (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT "");',
|
|
93
|
+
'CREATE TABLE items_fts_map (rowid INTEGER PRIMARY KEY AUTOINCREMENT, item_id TEXT, text TEXT);',
|
|
94
|
+
'CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);',
|
|
95
|
+
'CREATE VIRTUAL TABLE items_fts USING fts5(text);',
|
|
96
|
+
].join('\n');
|
|
97
|
+
const REBUILD = "INSERT INTO items_fts(items_fts) VALUES('delete-all'); INSERT INTO items_fts(rowid, text) SELECT rowid, text FROM items_fts_map;";
|
|
98
|
+
|
|
99
|
+
test('🔴 the virtual table and its shadows stay out, the MAP and the sequence come in', () => {
|
|
100
|
+
expect(backupTables(MASTER)).toEqual(['items', 'items_fts_map', 'meta', 'sqlite_sequence']);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
const fake = (rowsFor: (table: string) => string | null): { run: Wrangler; exported: string[]; seen: string[][] } => {
|
|
104
|
+
const exported: string[] = [];
|
|
105
|
+
const seen: string[][] = [];
|
|
106
|
+
const run: Wrangler = (args) => {
|
|
107
|
+
seen.push(args);
|
|
108
|
+
if (args[1] === 'execute') return { code: 0, stdout: `noise\n${JSON.stringify([{ results: MASTER }])}`, stderr: '' };
|
|
109
|
+
const table = args[args.indexOf('--table') + 1] as string;
|
|
110
|
+
const out = args[args.indexOf('--output') + 1] as string;
|
|
111
|
+
const rows = rowsFor(table);
|
|
112
|
+
if (rows === null) return { code: 1, stdout: '', stderr: `✘ export of ${table} failed` };
|
|
113
|
+
exported.push(table);
|
|
114
|
+
writeFileSync(out, rows);
|
|
115
|
+
return { code: 0, stdout: '', stderr: '' };
|
|
116
|
+
};
|
|
117
|
+
return { run, exported, seen };
|
|
118
|
+
};
|
|
119
|
+
const pull = (dir: string, run: Wrangler) =>
|
|
120
|
+
pullD1ToSqlite({
|
|
121
|
+
destination: join(dir, 'pulled.sqlite'),
|
|
122
|
+
database: 'collections',
|
|
123
|
+
schemaSql: SCHEMA,
|
|
124
|
+
requiredTable: 'items',
|
|
125
|
+
afterLoadSql: REBUILD,
|
|
126
|
+
wrangler: run,
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test('a pull is a real, searchable sqlite file built from the schema and the rows, of the named database', async () => {
|
|
130
|
+
const dir = scratch();
|
|
131
|
+
const { run, exported, seen } = fake((table) =>
|
|
132
|
+
table === 'items'
|
|
133
|
+
? "INSERT INTO items (id, name) VALUES ('itm_1', 'Travels 5388.jpg');"
|
|
134
|
+
: table === 'items_fts_map'
|
|
135
|
+
? "INSERT INTO items_fts_map (rowid, item_id, text) VALUES (1, 'itm_1', 'travels 5388');"
|
|
136
|
+
: '',
|
|
137
|
+
);
|
|
138
|
+
await pull(dir, run);
|
|
139
|
+
expect(exported).toEqual(['items', 'items_fts_map', 'meta', 'sqlite_sequence']);
|
|
140
|
+
expect(seen.every((args) => args[2] === 'collections')).toBe(true);
|
|
141
|
+
const db = new Database(join(dir, 'pulled.sqlite'), { readonly: true });
|
|
142
|
+
const hits = db.query("select count(*) as n from items_fts where items_fts match '5388'").get() as { n: number };
|
|
143
|
+
db.close();
|
|
144
|
+
expect(hits.n).toBe(1);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test('🔴 a database without the required table is refused as the wrong one', async () => {
|
|
148
|
+
const dir = scratch();
|
|
149
|
+
const { run } = fake(() => '');
|
|
150
|
+
await expect(
|
|
151
|
+
pullD1ToSqlite({ destination: join(dir, 'x.sqlite'), database: 'x', schemaSql: SCHEMA, requiredTable: 'tracks', wrangler: run }),
|
|
152
|
+
).rejects.toThrow(/lists no `tracks` table/);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('🔴 one table failing to export THROWS — never a snapshot missing that table', async () => {
|
|
156
|
+
const dir = scratch();
|
|
157
|
+
const { run } = fake((table) => (table === 'meta' ? null : "INSERT INTO items (id) VALUES ('x');"));
|
|
158
|
+
await expect(pull(dir, run)).rejects.toThrow(/--table meta failed/);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test('🔴 an export with no rows at all THROWS — never an empty library dated today', async () => {
|
|
162
|
+
const dir = scratch();
|
|
163
|
+
const { run } = fake(() => '');
|
|
164
|
+
await expect(pull(dir, run)).rejects.toThrow(/no rows/);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* After a Worker cutover, the Mac's nightly backup must snapshot the LIVE rows — D1 — and not
|
|
3
|
+
* the Mac's frozen file. The three pieces of that, which `apps/collections` wrote first
|
|
4
|
+
* (2026-09-19/22) and `apps/music` needed next, so they live here once:
|
|
5
|
+
*
|
|
6
|
+
* · {@link chooseBackupSource} — which side holds the live rows, asked of the production
|
|
7
|
+
* hostname's `/healthz` rather than of a config flag, and REFUSING when it cannot tell;
|
|
8
|
+
* · {@link servedRuntime} — that probe;
|
|
9
|
+
* · {@link pullD1ToSqlite} — the live database pulled down into a real SQLite file, table by
|
|
10
|
+
* table ({@link backupTables}), so the app's existing whole-file snapshot path is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* ## 🔴 Why the production hostname and not `cursed.app.runtime`
|
|
13
|
+
*
|
|
14
|
+
* An app declares `runtime: "worker"` when it is PORTED, which can be months before a single
|
|
15
|
+
* row moves. Keying on the marker points the nightly backup at an empty D1 the day the port
|
|
16
|
+
* lands. `/healthz` publishes `runtime` for exactly this question — ask the hostname who
|
|
17
|
+
* answered. And an unreadable answer REFUSES: guessing the Mac snapshots a frozen file after
|
|
18
|
+
* the cutover (a GREEN backup of nothing current — the worst failure a backup has), guessing
|
|
19
|
+
* D1 snapshots a stale copy while the Mac is live.
|
|
20
|
+
*
|
|
21
|
+
* ## 🔴 Why table by table, and through `wrangler`
|
|
22
|
+
*
|
|
23
|
+
* `wrangler d1 export` REFUSES a whole database holding a virtual table — *"cannot export
|
|
24
|
+
* databases with Virtual Tables (fts5)"*, measured on collections' first post-cutover night —
|
|
25
|
+
* and a per-table export is allowed. `wrangler` rather than the REST API because the export
|
|
26
|
+
* is typed SQL: the REST API's JSON cannot tell the REAL `5.0` from the INTEGER `5`
|
|
27
|
+
* (`./http.ts` has the measurement), and a restore must put back what was there.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { Database } from 'bun:sqlite';
|
|
31
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
32
|
+
|
|
33
|
+
export type BackupSourceChoice = { kind: 'file' | 'd1' | 'refuse'; why: string };
|
|
34
|
+
|
|
35
|
+
export interface BackupSourceInput {
|
|
36
|
+
/** `--from-file`: the operator overriding all of this on purpose. */
|
|
37
|
+
fromFileFlag: boolean;
|
|
38
|
+
/** `package.json`'s `cursed.app.runtime`. NOT sufficient on its own — see the header. */
|
|
39
|
+
runtimeMarker: string | undefined;
|
|
40
|
+
/** What the PRODUCTION hostname's `/healthz` said its `runtime` was; `null` = no usable answer. */
|
|
41
|
+
servedRuntime: string | null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Which side a backup must read. Pure, so every branch is testable without a network. */
|
|
45
|
+
export function chooseBackupSource(input: BackupSourceInput): BackupSourceChoice {
|
|
46
|
+
if (input.fromFileFlag) {
|
|
47
|
+
return { kind: 'file', why: "--from-file was given: snapshotting this Mac's sqlite on purpose" };
|
|
48
|
+
}
|
|
49
|
+
if (input.runtimeMarker !== 'worker') {
|
|
50
|
+
return { kind: 'file', why: 'this app does not declare a worker runtime, so the Mac holds the live rows' };
|
|
51
|
+
}
|
|
52
|
+
if (input.servedRuntime === null) {
|
|
53
|
+
return {
|
|
54
|
+
kind: 'refuse',
|
|
55
|
+
why:
|
|
56
|
+
"could not read `runtime` from the production hostname's /healthz, so which side holds the live " +
|
|
57
|
+
'rows is unknown. Guessing the Mac would snapshot a database that may have been frozen at the ' +
|
|
58
|
+
'cutover; guessing D1 would snapshot a stale copy while the Mac is still serving. Neither is a ' +
|
|
59
|
+
'backup. Fix the probe, or run `bun run backup -- --from-file` if you know the Mac is live.',
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
if (input.servedRuntime === 'worker') {
|
|
63
|
+
return { kind: 'd1', why: 'the production hostname is served by the Worker, so the live rows are in D1' };
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
kind: 'file',
|
|
67
|
+
why: `the production hostname is still served by '${input.servedRuntime}', so the Mac holds the live rows`,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* What `/healthz` says `runtime` is at `url`, or `null` if no usable answer came back — which
|
|
73
|
+
* means UNKNOWN and is never conflated with "not the Worker". Three attempts, because one blip
|
|
74
|
+
* must not cost a night's backup; a short timeout, because a hung fetch leaves launchd with a
|
|
75
|
+
* job that never exits.
|
|
76
|
+
*/
|
|
77
|
+
export async function servedRuntime(
|
|
78
|
+
url: string,
|
|
79
|
+
{ attempts = 3, sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms)) } = {},
|
|
80
|
+
): Promise<string | null> {
|
|
81
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
82
|
+
try {
|
|
83
|
+
const response = await fetch(`${url.replace(/\/+$/, '')}/healthz`, {
|
|
84
|
+
signal: AbortSignal.timeout(10_000),
|
|
85
|
+
headers: { 'cache-control': 'no-cache' },
|
|
86
|
+
});
|
|
87
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
88
|
+
const body = (await response.json()) as { runtime?: unknown };
|
|
89
|
+
// A body with no `runtime` is not an answer.
|
|
90
|
+
if (typeof body.runtime === 'string' && body.runtime) return body.runtime;
|
|
91
|
+
return null;
|
|
92
|
+
} catch {
|
|
93
|
+
if (attempt === attempts) return null;
|
|
94
|
+
await sleep(2000 * attempt);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** One `wrangler` invocation, as {@link pullD1ToSqlite} needs it. Injected so a spec runs none. */
|
|
101
|
+
export type Wrangler = (args: string[]) => { code: number; stdout: string; stderr: string };
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The tables a snapshot carries, from D1's own `sqlite_master` rows — every real table, and
|
|
105
|
+
* none of what cannot or must not be re-inserted: a virtual table and its shadows (`_config`,
|
|
106
|
+
* `_data`, `_docsize`, `_idx`, `_content` — an inverted index, not rows), `_cf_*`
|
|
107
|
+
* (Cloudflare's bookkeeping), and every `sqlite_*` table except `sqlite_sequence`, whose
|
|
108
|
+
* AUTOINCREMENT high-water marks belong in a restore.
|
|
109
|
+
*/
|
|
110
|
+
export function backupTables(rows: ReadonlyArray<{ name: string; sql: string | null }>): string[] {
|
|
111
|
+
const virtual = rows.filter((r) => /^\s*CREATE\s+VIRTUAL\s+TABLE/i.test(r.sql ?? '')).map((r) => r.name);
|
|
112
|
+
const shadow = new Set(
|
|
113
|
+
virtual.flatMap((vt) => ['config', 'data', 'docsize', 'idx', 'content'].map((suffix) => `${vt}_${suffix}`)),
|
|
114
|
+
);
|
|
115
|
+
return rows
|
|
116
|
+
.map((r) => r.name)
|
|
117
|
+
.filter((name) => !virtual.includes(name) && !shadow.has(name))
|
|
118
|
+
.filter((name) => !name.startsWith('_cf_'))
|
|
119
|
+
.filter((name) => name === 'sqlite_sequence' || !name.startsWith('sqlite_'))
|
|
120
|
+
.sort();
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export interface PullD1Options {
|
|
124
|
+
/** The SQLite file to create. */
|
|
125
|
+
destination: string;
|
|
126
|
+
/** The D1 database's NAME as `wrangler.jsonc` declares it at the top level. */
|
|
127
|
+
database: string;
|
|
128
|
+
/** The whole schema — normally the app's `db/schema.sql`. */
|
|
129
|
+
schemaSql: string;
|
|
130
|
+
/** A table that must be listed, or the pull is refused as the wrong database. */
|
|
131
|
+
requiredTable: string;
|
|
132
|
+
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
133
|
+
afterLoadSql?: string;
|
|
134
|
+
wrangler: Wrangler;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Pull the live D1 database into a real SQLite file.
|
|
139
|
+
*
|
|
140
|
+
* 🔴 It THROWS rather than returning an empty database. Every layer beneath treats a readable
|
|
141
|
+
* SQLite file as success, so an export that produced nothing would sail through the
|
|
142
|
+
* magic-byte check and `integrity_check` and land on the shelf with today's date on it.
|
|
143
|
+
*/
|
|
144
|
+
export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
|
|
145
|
+
const { destination, database, wrangler } = options;
|
|
146
|
+
const listed = wrangler([
|
|
147
|
+
'd1', 'execute', database, '--remote', '--env=', '--json',
|
|
148
|
+
'--command', "select name, sql from sqlite_master where type = 'table'",
|
|
149
|
+
]);
|
|
150
|
+
if (listed.code !== 0) throw new Error(`could not list D1's tables: ${listed.stderr.trim() || listed.stdout.trim()}`);
|
|
151
|
+
const json = listed.stdout.slice(listed.stdout.indexOf('['));
|
|
152
|
+
const rows = (JSON.parse(json) as Array<{ results: Array<{ name: string; sql: string | null }> }>)[0]?.results ?? [];
|
|
153
|
+
const tables = backupTables(rows);
|
|
154
|
+
if (!tables.includes(options.requiredTable)) {
|
|
155
|
+
throw new Error(
|
|
156
|
+
`D1 lists no \`${options.requiredTable}\` table (saw: ${rows.map((r) => r.name).join(', ') || 'nothing'})`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
let sql = '';
|
|
161
|
+
for (const table of tables) {
|
|
162
|
+
const dump = `${destination}.${table}.sql`;
|
|
163
|
+
const exported = wrangler(['d1', 'export', database, '--remote', '--env=', '--table', table, '--no-schema', '--output', dump, '-y']);
|
|
164
|
+
if (exported.code !== 0) {
|
|
165
|
+
throw new Error(`wrangler d1 export --table ${table} failed: ${exported.stderr.trim() || exported.stdout.trim()}`);
|
|
166
|
+
}
|
|
167
|
+
if (!existsSync(dump)) throw new Error(`wrangler d1 export --table ${table} reported success and wrote no file at ${dump}`);
|
|
168
|
+
sql += `${readFileSync(dump, 'utf8')}\n`;
|
|
169
|
+
}
|
|
170
|
+
if (!/INSERT INTO/i.test(sql)) {
|
|
171
|
+
throw new Error('the D1 export contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const db = new Database(destination, { create: true });
|
|
175
|
+
try {
|
|
176
|
+
// `exec`, not `run`: each is many statements and `run` would execute only the first,
|
|
177
|
+
// leaving a file with a schema and no rows that looks perfectly valid.
|
|
178
|
+
db.exec(options.schemaSql);
|
|
179
|
+
db.exec(sql);
|
|
180
|
+
if (options.afterLoadSql) db.exec(options.afterLoadSql);
|
|
181
|
+
} finally {
|
|
182
|
+
db.close();
|
|
183
|
+
}
|
|
184
|
+
}
|