@coywolf/emdash 0.0.0-stage → 0.35.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +203 -2
  3. package/docs/README.md +45 -0
  4. package/docs/ai.md +39 -0
  5. package/docs/backups.md +88 -0
  6. package/docs/breadcrumbs.md +59 -0
  7. package/docs/code-blocks.md +21 -0
  8. package/docs/content-urls.md +41 -0
  9. package/docs/custom-blocks.md +27 -0
  10. package/docs/discovery.md +32 -0
  11. package/docs/files.md +44 -0
  12. package/docs/form-uploads.md +59 -0
  13. package/docs/headings.md +37 -0
  14. package/docs/images.md +141 -0
  15. package/docs/internals.md +13 -0
  16. package/docs/links.md +34 -0
  17. package/docs/performance.md +72 -0
  18. package/docs/redirects.md +84 -0
  19. package/docs/releasing.md +21 -0
  20. package/docs/reviews.md +89 -0
  21. package/docs/robots.md +81 -0
  22. package/docs/schema.md +79 -0
  23. package/docs/search.md +114 -0
  24. package/docs/videos.md +48 -0
  25. package/docs/wordpress-import.md +198 -0
  26. package/package.json +79 -5
  27. package/src/admin/admin-css.ts +57 -0
  28. package/src/admin/ai.tsx +915 -0
  29. package/src/admin/backups.tsx +613 -0
  30. package/src/admin/breadcrumbs.tsx +542 -0
  31. package/src/admin/codeBlocks.tsx +204 -0
  32. package/src/admin/customBlocks.tsx +321 -0
  33. package/src/admin/dirty.ts +20 -0
  34. package/src/admin/discovery.tsx +463 -0
  35. package/src/admin/download.ts +28 -0
  36. package/src/admin/feature-switches.tsx +124 -0
  37. package/src/admin/features.tsx +142 -0
  38. package/src/admin/files.tsx +791 -0
  39. package/src/admin/formUploads.tsx +445 -0
  40. package/src/admin/guides.tsx +18 -0
  41. package/src/admin/headings.tsx +262 -0
  42. package/src/admin/images.tsx +825 -0
  43. package/src/admin/index.tsx +62 -0
  44. package/src/admin/links.tsx +980 -0
  45. package/src/admin/performance.tsx +330 -0
  46. package/src/admin/preview.tsx +63 -0
  47. package/src/admin/redirects-removed.tsx +266 -0
  48. package/src/admin/redirects.tsx +470 -0
  49. package/src/admin/reviews.tsx +232 -0
  50. package/src/admin/robots-bots.tsx +647 -0
  51. package/src/admin/robots-shared.tsx +109 -0
  52. package/src/admin/robots-wizard.tsx +748 -0
  53. package/src/admin/robots.tsx +911 -0
  54. package/src/admin/save-bar.tsx +134 -0
  55. package/src/admin/schema.tsx +1187 -0
  56. package/src/admin/search.tsx +355 -0
  57. package/src/admin/settings-ui.tsx +87 -0
  58. package/src/admin/trash-prompt-core.ts +135 -0
  59. package/src/admin/trash-prompt.tsx +156 -0
  60. package/src/admin/usage.tsx +301 -0
  61. package/src/admin/videos-display.tsx +248 -0
  62. package/src/admin/videos.tsx +1202 -0
  63. package/src/admin/wpImport.tsx +1516 -0
  64. package/src/ai/entities.ts +59 -0
  65. package/src/ai/export.ts +61 -0
  66. package/src/ai/logic.ts +606 -0
  67. package/src/ai/pack.ts +121 -0
  68. package/src/ai/prompts.ts +87 -0
  69. package/src/ai/providers.ts +276 -0
  70. package/src/ai/routes.ts +359 -0
  71. package/src/ai/store.ts +301 -0
  72. package/src/ai/wikidata.ts +87 -0
  73. package/src/ai/worker.ts +530 -0
  74. package/src/astro/astro-modules.d.ts +6 -0
  75. package/src/astro/breadcrumbs/Breadcrumbs.astro +129 -0
  76. package/src/astro/breadcrumbs/BreadcrumbsBlock.astro +12 -0
  77. package/src/astro/codeBlocks/CodeBlock.astro +62 -0
  78. package/src/astro/codeBlocks/EmDashCode.astro +61 -0
  79. package/src/astro/customBlocks/CustomBlock.astro +64 -0
  80. package/src/astro/customBlocks/Details.astro +8 -0
  81. package/src/astro/customBlocks/Disclosure.astro +8 -0
  82. package/src/astro/customBlocks/Note.astro +8 -0
  83. package/src/astro/customBlocks/Podcast.astro +8 -0
  84. package/src/astro/customBlocks/Quote.astro +8 -0
  85. package/src/astro/customBlocks/Testimonial.astro +8 -0
  86. package/src/astro/files/FileCard.astro +103 -0
  87. package/src/astro/headings/HeadingBlock.astro +129 -0
  88. package/src/astro/headings/TableOfContents.astro +45 -0
  89. package/src/astro/headings/request.ts +19 -0
  90. package/src/astro/images/ResponsiveImage.astro +60 -0
  91. package/src/astro/index.ts +84 -0
  92. package/src/astro/reviews/CoywolfReview.astro +27 -0
  93. package/src/astro/reviews/Review.astro +49 -0
  94. package/src/astro/search/SearchBox.astro +61 -0
  95. package/src/astro/search/SearchBoxForm.astro +509 -0
  96. package/src/astro/tables/CaptionedTable.astro +72 -0
  97. package/src/astro/videos/CoywolfVideo.astro +448 -0
  98. package/src/backups/download.ts +112 -0
  99. package/src/backups/dump.ts +288 -0
  100. package/src/backups/module.ts +276 -0
  101. package/src/backups/pack.ts +33 -0
  102. package/src/backups/restore.ts +209 -0
  103. package/src/backups/store.ts +409 -0
  104. package/src/breadcrumbs/pack.ts +113 -0
  105. package/src/breadcrumbs/settings.ts +102 -0
  106. package/src/breadcrumbs/stamp.ts +40 -0
  107. package/src/breadcrumbs/trail.ts +123 -0
  108. package/src/codeBlocks/contrast.ts +179 -0
  109. package/src/codeBlocks/hljs.d.ts +5 -0
  110. package/src/codeBlocks/pack.ts +110 -0
  111. package/src/codeBlocks/render.ts +304 -0
  112. package/src/codeBlocks/settings.ts +18 -0
  113. package/src/codeBlocks/themes.generated.ts +304 -0
  114. package/src/codeBlocks/themes.ts +114 -0
  115. package/src/core/compose.ts +244 -0
  116. package/src/core/content-url.ts +888 -0
  117. package/src/core/csv.ts +119 -0
  118. package/src/core/d1-batch.ts +65 -0
  119. package/src/core/entry-docs.ts +175 -0
  120. package/src/core/features-module.ts +48 -0
  121. package/src/core/features.ts +392 -0
  122. package/src/core/generation.ts +88 -0
  123. package/src/core/guide-content.ts +166 -0
  124. package/src/core/module.ts +104 -0
  125. package/src/core/read-failures.ts +57 -0
  126. package/src/core/request-memo.ts +71 -0
  127. package/src/core/secrets.ts +83 -0
  128. package/src/core/settings-enhance.ts +74 -0
  129. package/src/core/site.ts +36 -0
  130. package/src/core/storage.ts +27 -0
  131. package/src/customBlocks/pack.ts +260 -0
  132. package/src/customBlocks/render.ts +424 -0
  133. package/src/customBlocks/rich.ts +313 -0
  134. package/src/customBlocks/settings.ts +46 -0
  135. package/src/discovery/build.ts +252 -0
  136. package/src/discovery/feed-build.ts +217 -0
  137. package/src/discovery/feed.ts +172 -0
  138. package/src/discovery/indexnow.ts +114 -0
  139. package/src/discovery/llms.ts +79 -0
  140. package/src/discovery/markdown.ts +294 -0
  141. package/src/discovery/memory-cache.ts +97 -0
  142. package/src/discovery/middleware.ts +186 -0
  143. package/src/discovery/module.ts +353 -0
  144. package/src/discovery/news.ts +99 -0
  145. package/src/discovery/pack.ts +58 -0
  146. package/src/discovery/settings.ts +91 -0
  147. package/src/discovery/urls.ts +35 -0
  148. package/src/env.d.ts +10 -0
  149. package/src/files/card.ts +231 -0
  150. package/src/files/counts.ts +60 -0
  151. package/src/files/export.ts +44 -0
  152. package/src/files/format.ts +163 -0
  153. package/src/files/module.ts +596 -0
  154. package/src/files/pack.ts +92 -0
  155. package/src/files/r2-rules.ts +41 -0
  156. package/src/files/range.ts +50 -0
  157. package/src/files/s3.ts +114 -0
  158. package/src/files/serve.ts +141 -0
  159. package/src/files/sigv4.ts +160 -0
  160. package/src/files/site.ts +137 -0
  161. package/src/files/walker.ts +58 -0
  162. package/src/formUploads/lib.ts +266 -0
  163. package/src/formUploads/pack.ts +55 -0
  164. package/src/formUploads/wrap.ts +592 -0
  165. package/src/forms.ts +16 -0
  166. package/src/headings/pack.ts +154 -0
  167. package/src/headings/render.ts +139 -0
  168. package/src/headings/settings.ts +117 -0
  169. package/src/headings/slug.ts +68 -0
  170. package/src/headings/stamp.ts +142 -0
  171. package/src/headings/test-hooks.mjs +14 -0
  172. package/src/headings/toc.ts +48 -0
  173. package/src/images/cloudflare.ts +329 -0
  174. package/src/images/lib.ts +221 -0
  175. package/src/images/module.ts +372 -0
  176. package/src/images/pack.ts +804 -0
  177. package/src/images/pending.ts +37 -0
  178. package/src/images/settings.ts +48 -0
  179. package/src/images/setup.ts +192 -0
  180. package/src/images/variants-estimate.ts +162 -0
  181. package/src/images/variants-store.ts +475 -0
  182. package/src/images/variants.ts +311 -0
  183. package/src/index.ts +143 -0
  184. package/src/links/check.ts +101 -0
  185. package/src/links/classify.ts +270 -0
  186. package/src/links/export.ts +59 -0
  187. package/src/links/module.ts +652 -0
  188. package/src/links/pack.ts +43 -0
  189. package/src/links/pt.ts +325 -0
  190. package/src/links/store.ts +704 -0
  191. package/src/middleware.ts +287 -0
  192. package/src/modules.ts +53 -0
  193. package/src/options.ts +66 -0
  194. package/src/pageCache/lib.ts +274 -0
  195. package/src/pageCache/pack.ts +240 -0
  196. package/src/pageCache/usage-core.ts +152 -0
  197. package/src/pageCache/usage.ts +262 -0
  198. package/src/pageCache/warm.ts +722 -0
  199. package/src/redirects/middleware.ts +169 -0
  200. package/src/redirects/module.ts +124 -0
  201. package/src/redirects/pack.ts +89 -0
  202. package/src/redirects/removed-core.ts +151 -0
  203. package/src/redirects/removed.ts +293 -0
  204. package/src/redirects/rules.ts +355 -0
  205. package/src/redirects/slug-changes.ts +207 -0
  206. package/src/redirects/slug-core.ts +49 -0
  207. package/src/redirects/transfer.ts +92 -0
  208. package/src/reviews/lib.ts +624 -0
  209. package/src/reviews/pack.ts +135 -0
  210. package/src/reviews/schema.ts +128 -0
  211. package/src/reviews/settings.ts +26 -0
  212. package/src/robots/bots.ts +321 -0
  213. package/src/robots/data/bots.json +696 -0
  214. package/src/robots/data/verified.json +582 -0
  215. package/src/robots/directory.ts +26 -0
  216. package/src/robots/explain.ts +191 -0
  217. package/src/robots/guided.ts +300 -0
  218. package/src/robots/importer.ts +263 -0
  219. package/src/robots/middleware.ts +151 -0
  220. package/src/robots/module.ts +446 -0
  221. package/src/robots/pack.ts +69 -0
  222. package/src/robots/radar.ts +129 -0
  223. package/src/robots/rep.ts +362 -0
  224. package/src/robots/rules.ts +883 -0
  225. package/src/robots/transfer.ts +45 -0
  226. package/src/robots/validate.ts +524 -0
  227. package/src/schema/catalog.ts +146 -0
  228. package/src/schema/graph.ts +791 -0
  229. package/src/schema/module.ts +1138 -0
  230. package/src/schema/og-image.ts +100 -0
  231. package/src/schema/pack.ts +58 -0
  232. package/src/search/engine.ts +578 -0
  233. package/src/search/fallback.ts +80 -0
  234. package/src/search/live-cache.ts +116 -0
  235. package/src/search/live-client.ts +770 -0
  236. package/src/search/live-serve.ts +282 -0
  237. package/src/search/module.ts +261 -0
  238. package/src/search/pack.ts +57 -0
  239. package/src/search/query.ts +132 -0
  240. package/src/search/ratelimit-core.ts +118 -0
  241. package/src/search/ratelimit.ts +79 -0
  242. package/src/search/snippet.ts +135 -0
  243. package/src/search/title-index.ts +139 -0
  244. package/src/search/title-match.ts +192 -0
  245. package/src/shared.ts +71 -0
  246. package/src/version.ts +2 -0
  247. package/src/videos/date.ts +40 -0
  248. package/src/videos/engagement.ts +75 -0
  249. package/src/videos/lib.ts +679 -0
  250. package/src/videos/module.ts +1160 -0
  251. package/src/videos/pack.ts +187 -0
  252. package/src/videos/pause.ts +42 -0
  253. package/src/videos/poster.ts +324 -0
  254. package/src/videos/render.ts +336 -0
  255. package/src/videos/store.ts +551 -0
  256. package/src/videos/stream.ts +134 -0
  257. package/src/wpImport/convert.ts +702 -0
  258. package/src/wpImport/guests.ts +329 -0
  259. package/src/wpImport/gutenberg.ts +157 -0
  260. package/src/wpImport/markers.ts +66 -0
  261. package/src/wpImport/pack.ts +451 -0
  262. package/src/wpImport/parents.ts +202 -0
  263. package/src/wpImport/prepare.ts +592 -0
  264. package/src/wpImport/redirects.ts +358 -0
  265. package/src/wpImport/sources.ts +67 -0
  266. package/src/wpImport/stream.ts +160 -0
  267. package/src/wpImport/urls.ts +354 -0
@@ -0,0 +1,288 @@
1
+ /**
2
+ * Restorable SQL dump of a D1 database, built from inside the Worker.
3
+ *
4
+ * Same output order as scripts/backup/d1-backup.sh, so either kind of dump
5
+ * restores with scripts/backup/d1-restore.sh:
6
+ *
7
+ * 1. CREATE TABLE for every regular table
8
+ * 2. FTS5 search tables and the triggers that maintain them
9
+ * 3. INSERT rows, one statement per line (the search triggers re-index
10
+ * content rows as they load)
11
+ * 4. indexes and the remaining triggers
12
+ *
13
+ * FTS5 shadow tables are skipped: SQLite recreates them with the virtual table.
14
+ *
15
+ * D1 rejects any statement over 100 KB (SQLITE_TOOBIG), but the site writes
16
+ * rows with bound parameters, so a long post can be bigger than that. A row
17
+ * whose INSERT would be too long is written with its long text values cut
18
+ * short, followed by UPDATEs that append the rest in pieces under the limit.
19
+ */
20
+
21
+ interface MasterRow {
22
+ type: string;
23
+ name: string;
24
+ tbl_name: string;
25
+ sql: string | null;
26
+ }
27
+
28
+ const PAGE_SIZE = 500;
29
+ /**
30
+ * About how much a read of rows should return. D1 refuses query results over 50 MB (from
31
+ * November 4, 2026); 500 rows of a table with very large rows (long posts, revisions) could
32
+ * pass that, so each read is sized from the bytes the previous one returned.
33
+ */
34
+ export const PAGE_TARGET_BYTES = 8 * 1024 * 1024;
35
+ /** How D1 words a result over its size limit (its message has said "too large", "exceeds" or "limit" with a size). */
36
+ const RESULT_TOO_LARGE = /too large|too big|exceed|size limit|limit.*(bytes|MB)|SQLITE_TOOBIG/i;
37
+
38
+ /** Rows to ask for next, from the size of the rows just read. */
39
+ export function nextPageSize(pageBytes: number, rows: number): number {
40
+ if (rows <= 0) return PAGE_SIZE;
41
+ const perRow = Math.max(1, pageBytes / rows);
42
+ return Math.max(1, Math.min(PAGE_SIZE, Math.floor(PAGE_TARGET_BYTES / perRow)));
43
+ }
44
+ const SEARCH_PREFIX = "_emdash_fts_";
45
+ /** Longest statement written, in UTF-8 bytes; D1's limit is 100,000. */
46
+ export const MAX_STATEMENT_BYTES = 90_000;
47
+ /** Most UTF-8 bytes of escaped text per piece of a split value. */
48
+ const PIECE_BYTES = 60_000;
49
+ /** In a row too long for one INSERT, text values longer than this are appended separately. */
50
+ const SHORT_BYTES = 2_000;
51
+ /**
52
+ * Line breaks are written as '||char(10)||', two operators each, and SQLite
53
+ * refuses expressions deeper than 1,000: at most this many per statement.
54
+ */
55
+ const MAX_BREAKS = 400;
56
+ const breaks = (s: string) => s.match(/[\r\n]/g)?.length ?? 0;
57
+ /** Option rows that the plugin rebuilds on demand; not worth backing up. */
58
+ const SKIP_OPTION = /^plugin:coywolf-pack:cache:/;
59
+
60
+ function isInternal(name: string): boolean {
61
+ return name.startsWith("sqlite_") || name.startsWith("_cf_");
62
+ }
63
+
64
+ function quoteIdent(name: string): string {
65
+ return `"${name.replace(/"/g, '""')}"`;
66
+ }
67
+
68
+ /** SQL literal for a D1 value, keeping every statement on one line (like sqlite3 .dump). */
69
+ export function sqlLiteral(value: unknown): string {
70
+ if (value === null || value === undefined) return "NULL";
71
+ if (typeof value === "number") return Number.isFinite(value) ? String(value) : "NULL";
72
+ if (typeof value === "bigint") return value.toString();
73
+ if (typeof value === "boolean") return value ? "1" : "0";
74
+ if (value instanceof ArrayBuffer || ArrayBuffer.isView(value) || Array.isArray(value)) {
75
+ const bytes =
76
+ value instanceof ArrayBuffer
77
+ ? new Uint8Array(value)
78
+ : Array.isArray(value)
79
+ ? Uint8Array.from(value as number[])
80
+ : new Uint8Array((value as ArrayBufferView).buffer, (value as ArrayBufferView).byteOffset, (value as ArrayBufferView).byteLength);
81
+ let hex = "";
82
+ for (const b of bytes) hex += b.toString(16).padStart(2, "0");
83
+ return `X'${hex}'`;
84
+ }
85
+ // Newlines become ||char(10)|| so each INSERT stays on one line.
86
+ return `'${escapeText(String(value))}'`;
87
+ }
88
+
89
+ function escapeText(text: string): string {
90
+ return text.replace(/'/g, "''").replace(/\r/g, "'||char(13)||'").replace(/\n/g, "'||char(10)||'");
91
+ }
92
+
93
+ const utf8 = new TextEncoder();
94
+ const bytes = (s: string) => utf8.encode(s).length;
95
+
96
+ /** Split text into pieces whose escaped form stays under PIECE_BYTES (never inside a surrogate pair). */
97
+ export function splitText(text: string): string[] {
98
+ const pieces: string[] = [];
99
+ let start = 0;
100
+ let size = 0;
101
+ let lines = 0;
102
+ for (let i = 0; i < text.length; i++) {
103
+ const code = text.charCodeAt(i);
104
+ const pair = code >= 0xd800 && code <= 0xdbff && i + 1 < text.length;
105
+ const char = pair ? text.slice(i, i + 2) : text[i];
106
+ const cost = bytes(escapeText(char));
107
+ const isBreak = char === "\n" || char === "\r";
108
+ if ((size + cost > PIECE_BYTES || (isBreak && lines >= MAX_BREAKS)) && i > start) {
109
+ pieces.push(text.slice(start, i));
110
+ start = i;
111
+ size = 0;
112
+ lines = 0;
113
+ }
114
+ size += cost;
115
+ if (isBreak) lines++;
116
+ if (pair) i++;
117
+ }
118
+ pieces.push(text.slice(start));
119
+ return pieces;
120
+ }
121
+
122
+ /**
123
+ * One row as statements: a single INSERT when it fits, otherwise an INSERT
124
+ * with each long text value's first piece, then UPDATEs appending the rest.
125
+ * `key` names the columns that identify the row (primary key, or rowid).
126
+ */
127
+ export function rowStatements(table: string, row: Record<string, unknown>, key: Record<string, unknown>): string[] {
128
+ const columns = Object.keys(row).filter((c) => c !== ROWID);
129
+ const insert = (values: Record<string, unknown>) =>
130
+ `INSERT INTO ${table} (${columns.map(quoteIdent).join(",")}) VALUES(${columns.map((c) => sqlLiteral(values[c])).join(",")});`;
131
+ const whole = insert(row);
132
+ const lineCount = columns.reduce((n, c) => n + (typeof row[c] === "string" ? breaks(row[c] as string) : 0), 0);
133
+ if (bytes(whole) <= MAX_STATEMENT_BYTES && lineCount <= MAX_BREAKS) return [whole];
134
+
135
+ // Long text values start empty and are appended piece by piece, so the
136
+ // INSERT holds only short values however many long columns the row has.
137
+ const first: Record<string, unknown> = { ...row };
138
+ const rest: Array<[string, string]> = [];
139
+ for (const c of columns) {
140
+ const value = row[c];
141
+ if (typeof value !== "string" || (bytes(sqlLiteral(value)) <= SHORT_BYTES && breaks(value) <= 20)) continue;
142
+ first[c] = "";
143
+ for (const piece of splitText(value)) rest.push([c, piece]);
144
+ }
145
+ const where = Object.entries(key)
146
+ .map(([c, v]) => `${c === ROWID ? "rowid" : quoteIdent(c)}=${sqlLiteral(v)}`)
147
+ .join(" AND ");
148
+ return [
149
+ insert(first),
150
+ ...rest.map(([c, piece]) => `UPDATE ${table} SET ${quoteIdent(c)}=${quoteIdent(c)}||${sqlLiteral(piece)} WHERE ${where};`),
151
+ ];
152
+ }
153
+
154
+ /** Alias for rowid in dump queries of tables without a primary key. */
155
+ const ROWID = "__coywolf_rowid";
156
+
157
+ /**
158
+ * Parents before children. D1 may apply a large import in several batches, so
159
+ * PRAGMA defer_foreign_keys can't be relied on to cover forward references.
160
+ */
161
+ async function orderByForeignKeys(db: D1Database, tables: MasterRow[]): Promise<MasterRow[]> {
162
+ const names = new Set(tables.map((t) => t.name));
163
+ const parents = new Map<string, Set<string>>();
164
+ for (const t of tables) {
165
+ const { results } = await db.prepare(`PRAGMA foreign_key_list(${quoteIdent(t.name)})`).all<{ table: string }>();
166
+ parents.set(t.name, new Set(results.map((r) => r.table).filter((p) => p !== t.name && names.has(p))));
167
+ }
168
+ const ordered: MasterRow[] = [];
169
+ const done = new Set<string>();
170
+ const visiting = new Set<string>();
171
+ const byName = new Map(tables.map((t) => [t.name, t]));
172
+ const visit = (name: string) => {
173
+ if (done.has(name) || visiting.has(name)) return; // a cycle falls back to name order
174
+ visiting.add(name);
175
+ for (const parent of parents.get(name) ?? []) visit(parent);
176
+ visiting.delete(name);
177
+ done.add(name);
178
+ ordered.push(byName.get(name)!);
179
+ };
180
+ for (const t of tables) visit(t.name);
181
+ return ordered;
182
+ }
183
+
184
+ export interface DumpResult {
185
+ sql: string;
186
+ tables: number;
187
+ rows: number;
188
+ }
189
+
190
+ /** The whole dump as one string (tests, small databases). Backups stream dumpLines instead. */
191
+ export async function dumpDatabase(db: D1Database): Promise<DumpResult> {
192
+ const counts = { tables: 0, rows: 0 };
193
+ const out: string[] = [];
194
+ for await (const line of dumpLines(db, counts)) out.push(line);
195
+ return { sql: `${out.join("\n")}\n`, ...counts };
196
+ }
197
+
198
+ /** Columns that, when a table declares them, hide the real rowid from `rowid` in queries. */
199
+ const ROWID_NAMES = new Set(["rowid", "_rowid_", "oid"]);
200
+
201
+ /**
202
+ * The dump, one statement per item (no trailing newline), read a page of rows
203
+ * at a time so memory stays bounded however big the database is. `counts` is
204
+ * filled in as it goes (complete once the iteration ends). Tables with a
205
+ * rowid are paged by rowid (`WHERE rowid > ?`), which stays fast on big
206
+ * tables where OFFSET rescans every earlier row; WITHOUT ROWID tables use OFFSET.
207
+ */
208
+ export async function* dumpLines(db: D1Database, counts: { tables: number; rows: number } = { tables: 0, rows: 0 }): AsyncGenerator<string> {
209
+ const { results: master } = await db
210
+ .prepare("SELECT type, name, tbl_name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name")
211
+ .all<MasterRow>();
212
+
213
+ const isSearch = (row: MasterRow) =>
214
+ row.name.startsWith(SEARCH_PREFIX) || row.tbl_name.startsWith(SEARCH_PREFIX) || (row.sql ?? "").includes(SEARCH_PREFIX);
215
+
216
+ const tables = master.filter((r) => r.type === "table" && !isInternal(r.name) && !r.name.startsWith(SEARCH_PREFIX));
217
+ const searchTables = master.filter(
218
+ (r) => r.type === "table" && r.name.startsWith(SEARCH_PREFIX) && /^CREATE VIRTUAL TABLE/i.test(r.sql ?? ""),
219
+ );
220
+ const searchTriggers = master.filter((r) => r.type === "trigger" && r.name.startsWith(SEARCH_PREFIX));
221
+ const indexes = master.filter((r) => r.type === "index" && !isInternal(r.name) && !isSearch(r));
222
+ const triggers = master.filter((r) => r.type === "trigger" && !isSearch(r));
223
+
224
+ const ordered = await orderByForeignKeys(db, tables);
225
+
226
+ const statement = (sql: string) => `${sql.trim().replace(/;$/, "")};`;
227
+ counts.tables = tables.length;
228
+ yield "PRAGMA defer_foreign_keys=TRUE;";
229
+
230
+ for (const t of tables) yield statement((t.sql ?? "").replace(/^CREATE TABLE /i, "CREATE TABLE IF NOT EXISTS "));
231
+ for (const t of searchTables) yield statement(t.sql ?? "");
232
+ for (const t of searchTriggers) yield statement(t.sql ?? "");
233
+
234
+ for (const t of ordered) {
235
+ const table = quoteIdent(t.name);
236
+ const { results: info } = await db.prepare(`PRAGMA table_info(${table})`).all<{ name: string; pk: number }>();
237
+ const withRowid = !/WITHOUT\s+ROWID/i.test(t.sql ?? "");
238
+ // A column named rowid (or oid) shadows the real one: page that table by OFFSET.
239
+ const byRowid = withRowid && !info.some((c) => ROWID_NAMES.has(c.name.toLowerCase()));
240
+ const pk = info.filter((c) => c.pk > 0).map((c) => c.name);
241
+ // Rows are identified by primary key for split values; tables without one by rowid.
242
+ const keyColumns = pk.length || !withRowid ? pk : [ROWID];
243
+ // The rowid alias is never written out (rowStatements skips it); it's the paging cursor.
244
+ const select = byRowid || (withRowid && !pk.length) ? `rowid AS ${ROWID}, *` : "*";
245
+ const order = withRowid ? " ORDER BY rowid" : "";
246
+ let after: unknown = null;
247
+ let offset = 0;
248
+ let limit = PAGE_SIZE;
249
+ for (;;) {
250
+ let results: Record<string, unknown>[];
251
+ try {
252
+ ({ results } = byRowid
253
+ ? await db
254
+ .prepare(`SELECT ${select} FROM ${table}${after === null ? "" : ` WHERE rowid > ?`}${order} LIMIT ?`)
255
+ .bind(...(after === null ? [limit] : [after, limit]))
256
+ .all<Record<string, unknown>>()
257
+ : await db
258
+ .prepare(`SELECT ${select} FROM ${table}${order} LIMIT ? OFFSET ?`)
259
+ .bind(limit, offset)
260
+ .all<Record<string, unknown>>());
261
+ } catch (error) {
262
+ // D1 refuses results over 50 MB (from November 4, 2026). Ask again for fewer rows, down to one.
263
+ // Any other error (an outage, a bad query) goes straight up instead of being retried.
264
+ if (limit === 1 || !RESULT_TOO_LARGE.test(String(error))) throw error;
265
+ limit = Math.max(1, Math.floor(limit / 4));
266
+ continue;
267
+ }
268
+ let pageBytes = 0;
269
+ for (const row of results) {
270
+ if (t.name === "options" && typeof row.name === "string" && SKIP_OPTION.test(row.name)) continue;
271
+ const key = Object.fromEntries(keyColumns.map((c) => [c, row[c]]));
272
+ for (const sql of rowStatements(table, row, key)) {
273
+ pageBytes += sql.length;
274
+ yield sql;
275
+ }
276
+ counts.rows++;
277
+ }
278
+ if (results.length < limit) break;
279
+ offset += results.length;
280
+ if (byRowid) after = results[results.length - 1][ROWID];
281
+ // Size the next read from this one: about PAGE_TARGET_BYTES, never more than PAGE_SIZE rows.
282
+ limit = nextPageSize(pageBytes, results.length);
283
+ }
284
+ }
285
+
286
+ for (const i of indexes) yield statement(i.sql ?? "");
287
+ for (const t of triggers) yield statement(t.sql ?? "");
288
+ }
@@ -0,0 +1,276 @@
1
+ /**
2
+ * Backups module: D1 dumps + R2 media mirror, with rewind (D1 Time Travel),
3
+ * undo, restore to a new database, and missing-media restore.
4
+ */
5
+ import { PluginRouteError, definePluginRoute } from "emdash";
6
+ import { z } from "zod";
7
+
8
+ import { type CronScheduler, type SettingsReader, parseInput, secret, workerEnv } from "../shared.js";
9
+ import { createDownloadLink } from "./download.js";
10
+ import { dumpLines } from "./dump.js";
11
+ import {
12
+ type RestoreConfig,
13
+ currentBookmark,
14
+ getUndo,
15
+ restoreMissingMedia,
16
+ restoreToNewDatabase,
17
+ rewind,
18
+ undoRewind,
19
+ } from "./restore.js";
20
+ import { FILE, type Manifest, STAMP, listBackups, makeStamp, mirrorBucket, mirrorMedia, pruneBackups, writeDumpStream, writeManifest } from "./store.js";
21
+
22
+ export interface BackupsOptions {
23
+ /** D1 binding of the site database. Default "DB". */
24
+ database?: string;
25
+ /** R2 binding of the media bucket. Default "MEDIA". */
26
+ media?: string;
27
+ /** R2 binding of the backup bucket (use a separate, private bucket). Default "BACKUPS". */
28
+ backups?: string;
29
+ /**
30
+ * R2 binding of the private form uploads bucket (Coywolf Pack → Private form
31
+ * uploads), mirrored to uploads/ in the backup bucket. Default "FORM_UPLOADS";
32
+ * skipped when the site has no such binding. false leaves uploads out.
33
+ */
34
+ uploads?: string | false;
35
+ /** Dump file name prefix (e.g. "mysite" -> mysite.sql.gz). Default "database". */
36
+ name?: string;
37
+ /**
38
+ * Restore via the Cloudflare API: the account and D1 database IDs, plus a
39
+ * Worker secret with an API token (Account → D1 → Edit). These live outside
40
+ * the database on purpose: a rewind rolls back plugin settings too.
41
+ */
42
+ restore?: { accountId: string; databaseId: string; tokenSecret?: string };
43
+ }
44
+
45
+ /**
46
+ * Schedule and retention, edited on the Backups page. Not in the plugin's
47
+ * settingsSchema (that holds secrets only), so every read supplies its default.
48
+ */
49
+ export const BACKUPS_DEFAULTS = { scheduled: false, retentionDays: 30, staleAfterHours: 36 } as const;
50
+
51
+ const backupsSettingsInput = z.object({
52
+ scheduled: z.boolean(),
53
+ retentionDays: z.number().int().min(1, "Keep backups for at least 1 day.").max(365, "Keep backups for at most 365 days."),
54
+ staleAfterHours: z.number().int().min(1, "Warn after at least 1 hour.").max(720, "Warn after at most 720 hours (30 days)."),
55
+ });
56
+
57
+ async function settings(ctx: SettingsReader) {
58
+ return {
59
+ scheduled: (await ctx.settings.get<boolean>("backupsScheduled")) ?? BACKUPS_DEFAULTS.scheduled,
60
+ retentionDays: (await ctx.settings.get<number>("backupsRetentionDays")) ?? BACKUPS_DEFAULTS.retentionDays,
61
+ staleAfterHours: (await ctx.settings.get<number>("backupsStaleAfterHours")) ?? BACKUPS_DEFAULTS.staleAfterHours,
62
+ };
63
+ }
64
+
65
+ /** Daily task for scheduled backups and retention. schedule() is an upsert, so it's safe to repeat. */
66
+ export const BACKUPS_TASK = "backups-daily";
67
+ export async function ensureBackupsTask(ctx: CronScheduler) {
68
+ await ctx.cron?.schedule(BACKUPS_TASK, { schedule: "@daily" });
69
+ }
70
+
71
+ const confirmed = (word: string) => z.literal(word, { message: `Type ${word} to confirm.` });
72
+
73
+ export function backupsModule(options: BackupsOptions) {
74
+ const name = options.name ?? "database";
75
+
76
+ async function bindings() {
77
+ const env = await workerEnv();
78
+ const db = env[options.database ?? "DB"] as D1Database | undefined;
79
+ const media = env[options.media ?? "MEDIA"] as R2Bucket | undefined;
80
+ const backups = env[options.backups ?? "BACKUPS"] as R2Bucket | undefined;
81
+ const uploads = options.uploads === false ? undefined : (env[options.uploads ?? "FORM_UPLOADS"] as R2Bucket | undefined);
82
+ const missing = [!db && "database", !media && "media", !backups && "backups"].filter(Boolean);
83
+ if (missing.length) throw PluginRouteError.badRequest(`Backups: missing ${missing.join(", ")} binding (see README).`);
84
+ const tokenName = options.restore?.tokenSecret ?? "BACKUPS_API_TOKEN";
85
+ const token = secret(env, tokenName);
86
+ const restore: RestoreConfig | null =
87
+ options.restore && token ? { token, accountId: options.restore.accountId, databaseId: options.restore.databaseId } : null;
88
+ const restoreStatus = restore
89
+ ? { enabled: true as const }
90
+ : {
91
+ enabled: false as const,
92
+ reason: options.restore
93
+ ? `Set the ${tokenName} Worker secret to an API token with D1 Edit permission.`
94
+ : "Add the backups.restore option (accountId, databaseId) in astro.config.mjs.",
95
+ };
96
+ return { db: db!, media: media!, backups: backups!, uploads, restore, restoreStatus };
97
+ }
98
+
99
+ async function readDump(backups: R2Bucket, stamp: string): Promise<{ manifest: Manifest; sql: string }> {
100
+ const manifestObject = await backups.get(`d1/${stamp}/manifest.json`);
101
+ if (!manifestObject) throw PluginRouteError.notFound("Backup not found");
102
+ const manifest = (await manifestObject.json()) as Manifest;
103
+ const dump = await backups.get(`d1/${stamp}/${manifest.file}`);
104
+ if (!dump) throw PluginRouteError.notFound("Backup file not found");
105
+ return { manifest, sql: await new Response(dump.body.pipeThrough(new DecompressionStream("gzip"))).text() };
106
+ }
107
+
108
+ async function runBackup(source: "admin" | "scheduled") {
109
+ const { db, media, backups, uploads, restore } = await bindings();
110
+ const stamp = makeStamp();
111
+ const timeTravelBookmark = restore ? await currentBookmark(restore).catch(() => undefined) : undefined;
112
+ const file = `${name}.sql.gz`;
113
+ // Streamed: rows are read a page at a time and gzipped straight into the bucket.
114
+ const counts = { tables: 0, rows: 0 };
115
+ const written = await writeDumpStream(backups, stamp, file, dumpLines(db, counts));
116
+ const extra = { database: name, source, timeTravelBookmark, tables: counts.tables, rows: counts.rows };
117
+ // Save the database first so a media problem can't cost the dump.
118
+ const manifest = await writeManifest(backups, stamp, file, written, extra);
119
+ let result = manifest;
120
+ try {
121
+ const mediaResult = await mirrorMedia(media, backups, stamp);
122
+ result = await writeManifest(backups, stamp, file, written, { ...extra, media: mediaResult });
123
+ } catch (error) {
124
+ console.error("coywolf backups: media mirror failed; the database backup was saved", error);
125
+ }
126
+ if (uploads) {
127
+ try {
128
+ const uploadsResult = await mirrorBucket(uploads, backups, stamp, "uploads");
129
+ result = await writeManifest(backups, stamp, file, written, { ...result, uploads: uploadsResult });
130
+ } catch (error) {
131
+ console.error("coywolf backups: form uploads mirror failed; the database backup was saved", error);
132
+ }
133
+ }
134
+ return result;
135
+ }
136
+
137
+ async function daily(ctx: SettingsReader & { log: { info(msg: string, data?: unknown): void } }) {
138
+ const s = await settings(ctx);
139
+ if (s.scheduled) {
140
+ const manifest = await runBackup("scheduled");
141
+ ctx.log.info("Scheduled backup complete", { stamp: manifest.stamp, bytes: manifest.bytes });
142
+ }
143
+ const { backups } = await bindings();
144
+ const pruned = await pruneBackups(backups, s.retentionDays);
145
+ if (pruned) ctx.log.info("Pruned old backups", { pruned });
146
+ }
147
+
148
+ const routes = {
149
+ "backups/list": {
150
+ permission: "plugins:manage" as const,
151
+ handler: async (ctx: SettingsReader & CronScheduler) => {
152
+ await ensureBackupsTask(ctx);
153
+ const { backups, restoreStatus } = await bindings();
154
+ return { items: await listBackups(backups), ...(await settings(ctx)), restore: restoreStatus, undo: await getUndo(backups) };
155
+ },
156
+ },
157
+
158
+ /** Schedule and retention (the Backups page's Settings section). */
159
+ "backups/settings/save": definePluginRoute({
160
+ permission: "plugins:manage",
161
+ methods: ["POST"],
162
+ request: { body: "json" },
163
+ handler: async (ctx) => {
164
+ const input = parseInput(backupsSettingsInput, ctx.input);
165
+ await ctx.settings.set("backupsScheduled", input.scheduled);
166
+ await ctx.settings.set("backupsRetentionDays", input.retentionDays);
167
+ await ctx.settings.set("backupsStaleAfterHours", input.staleAfterHours);
168
+ await ensureBackupsTask(ctx);
169
+ ctx.log.info("Backup settings saved", input);
170
+ return settings(ctx);
171
+ },
172
+ }),
173
+
174
+ "backups/run": definePluginRoute({
175
+ permission: "plugins:manage",
176
+ methods: ["POST"],
177
+ request: { body: "none" },
178
+ handler: async (ctx) => {
179
+ await ensureBackupsTask(ctx);
180
+ const manifest = await runBackup("admin");
181
+ ctx.log.info("Backup complete", { stamp: manifest.stamp, bytes: manifest.bytes });
182
+ const { backups } = await bindings();
183
+ await pruneBackups(backups, (await settings(ctx)).retentionDays);
184
+ return manifest;
185
+ },
186
+ }),
187
+
188
+ "backups/rewind": definePluginRoute({
189
+ permission: "plugins:manage",
190
+ methods: ["POST"],
191
+ request: { body: "json" },
192
+ handler: async (ctx) => {
193
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), confirm: confirmed("REWIND") }), ctx.input);
194
+ const { backups, restore } = await bindings();
195
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
196
+ const manifestObject = await backups.get(`d1/${input.stamp}/manifest.json`);
197
+ const manifest = manifestObject ? ((await manifestObject.json()) as Manifest) : undefined;
198
+ const undo = await rewind(restore, backups, { stamp: input.stamp, bookmark: manifest?.timeTravelBookmark });
199
+ ctx.log.warn("Database rewound", { to: input.stamp, undoBookmark: undo.bookmark });
200
+ return undo;
201
+ },
202
+ }),
203
+
204
+ "backups/undo": definePluginRoute({
205
+ permission: "plugins:manage",
206
+ methods: ["POST"],
207
+ request: { body: "json" },
208
+ handler: async (ctx) => {
209
+ parseInput(z.object({ confirm: confirmed("UNDO") }), ctx.input);
210
+ const { backups, restore } = await bindings();
211
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
212
+ const undo = await undoRewind(restore, backups);
213
+ ctx.log.warn("Rewind undone", { bookmark: undo.bookmark });
214
+ return undo;
215
+ },
216
+ }),
217
+
218
+ "backups/restore-new": definePluginRoute({
219
+ permission: "plugins:manage",
220
+ methods: ["POST"],
221
+ request: { body: "json" },
222
+ handler: async (ctx) => {
223
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), confirm: confirmed("RESTORE") }), ctx.input);
224
+ const { backups, restore } = await bindings();
225
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
226
+ const { sql } = await readDump(backups, input.stamp);
227
+ const result = await restoreToNewDatabase(restore, `${name}-restore-${input.stamp.toLowerCase()}`, sql);
228
+ ctx.log.warn("Backup restored to a new database", { stamp: input.stamp, database: result.databaseName });
229
+ return result;
230
+ },
231
+ }),
232
+
233
+ "backups/restore-media": definePluginRoute({
234
+ permission: "plugins:manage",
235
+ methods: ["POST"],
236
+ request: { body: "none" },
237
+ handler: async (ctx) => {
238
+ // Media first, then private form uploads, in batches; the admin calls again while anything is pending.
239
+ const { media, backups, uploads } = await bindings();
240
+ const result = await restoreMissingMedia(media, backups);
241
+ const fromUploads =
242
+ uploads && !result.pending ? await restoreMissingMedia(uploads, backups, 200, "uploads") : { restored: 0, checked: 0, pending: 0 };
243
+ const combined = {
244
+ restored: result.restored + fromUploads.restored,
245
+ checked: result.checked + fromUploads.checked,
246
+ // Uploads aren't looked at until media is done, so count them as pending meanwhile.
247
+ pending: result.pending + (uploads && result.pending ? 1 : fromUploads.pending),
248
+ media: result,
249
+ uploads: uploads ? fromUploads : undefined,
250
+ };
251
+ ctx.log.info("Missing media restored", combined);
252
+ return combined;
253
+ },
254
+ }),
255
+
256
+ /**
257
+ * A ten-minute link that streams one backup file (see ./download.ts).
258
+ * The file is a gzipped SQL dump that restores into any D1 database.
259
+ */
260
+ "backups/download-link": definePluginRoute({
261
+ permission: "plugins:manage",
262
+ methods: ["POST"],
263
+ request: { body: "json" },
264
+ handler: async (ctx) => {
265
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), file: z.string().regex(FILE) }), ctx.input);
266
+ const { backups } = await bindings();
267
+ const link = await createDownloadLink(backups, input.stamp, input.file);
268
+ if (!link) throw PluginRouteError.notFound("Backup file not found");
269
+ ctx.log.info("Backup download link created", { stamp: input.stamp, file: input.file });
270
+ return link;
271
+ },
272
+ }),
273
+ };
274
+
275
+ return { routes, daily };
276
+ }
@@ -0,0 +1,33 @@
1
+ import type { PluginContext } from "emdash";
2
+
3
+ import { registerFeatures } from "../core/features.js";
4
+ import type { PackMiddleware, PackModule } from "../core/module.js";
5
+ import { configureBackupDownloads, serveBackupDownload } from "./download.js";
6
+ import { BACKUPS_TASK, type BackupsOptions, backupsModule } from "./module.js";
7
+
8
+ const FEATURES = [
9
+ { id: "backups", label: "Backups", description: "Database and media backups, rewind, and restore.", default: true },
10
+ ];
11
+ registerFeatures(FEATURES);
12
+
13
+ export function backupsPack(options: BackupsOptions): PackModule {
14
+ // Module-level so the site middleware can find the backup bucket (createPlugin() runs at isolate start).
15
+ configureBackupDownloads(options);
16
+ const backups = backupsModule(options);
17
+ return {
18
+ id: "backups",
19
+ label: "Backups",
20
+ features: FEATURES,
21
+ routes: backups.routes,
22
+ tasks: [{ name: BACKUPS_TASK, schedule: "@daily", handler: (ctx: PluginContext) => backups.daily(ctx) }],
23
+ adminPages: [{ path: "/backups", label: "Backups", icon: "database" }],
24
+ widgets: [{ id: "backup-status", title: "Backups", size: "third" }],
25
+ };
26
+ }
27
+
28
+ /** Streams backup downloads at the links the Backups page creates (see ./download.ts). */
29
+ export const backupsDownloadMiddleware: PackMiddleware = {
30
+ module: "backups",
31
+ feature: "backups",
32
+ handle: (context, env) => serveBackupDownload(context.request, context.url, env),
33
+ };