@coywolf/emdash 0.0.0-stage → 0.35.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +201 -2
  3. package/package.json +70 -5
  4. package/src/admin/admin-css.ts +57 -0
  5. package/src/admin/ai.tsx +915 -0
  6. package/src/admin/backups.tsx +613 -0
  7. package/src/admin/breadcrumbs.tsx +542 -0
  8. package/src/admin/codeBlocks.tsx +204 -0
  9. package/src/admin/customBlocks.tsx +321 -0
  10. package/src/admin/dirty.ts +20 -0
  11. package/src/admin/discovery.tsx +463 -0
  12. package/src/admin/download.ts +28 -0
  13. package/src/admin/feature-switches.tsx +124 -0
  14. package/src/admin/features.tsx +142 -0
  15. package/src/admin/files.tsx +791 -0
  16. package/src/admin/formUploads.tsx +445 -0
  17. package/src/admin/guides.tsx +18 -0
  18. package/src/admin/headings.tsx +262 -0
  19. package/src/admin/images.tsx +825 -0
  20. package/src/admin/index.tsx +62 -0
  21. package/src/admin/links.tsx +980 -0
  22. package/src/admin/performance.tsx +330 -0
  23. package/src/admin/preview.tsx +63 -0
  24. package/src/admin/redirects-removed.tsx +266 -0
  25. package/src/admin/redirects.tsx +470 -0
  26. package/src/admin/reviews.tsx +232 -0
  27. package/src/admin/robots-bots.tsx +647 -0
  28. package/src/admin/robots-shared.tsx +109 -0
  29. package/src/admin/robots-wizard.tsx +748 -0
  30. package/src/admin/robots.tsx +911 -0
  31. package/src/admin/save-bar.tsx +134 -0
  32. package/src/admin/schema.tsx +1187 -0
  33. package/src/admin/search.tsx +355 -0
  34. package/src/admin/settings-ui.tsx +87 -0
  35. package/src/admin/trash-prompt-core.ts +135 -0
  36. package/src/admin/trash-prompt.tsx +156 -0
  37. package/src/admin/usage.tsx +301 -0
  38. package/src/admin/videos-display.tsx +248 -0
  39. package/src/admin/videos.tsx +1202 -0
  40. package/src/admin/wpImport.tsx +1516 -0
  41. package/src/ai/entities.ts +59 -0
  42. package/src/ai/export.ts +61 -0
  43. package/src/ai/logic.test.ts +187 -0
  44. package/src/ai/logic.ts +606 -0
  45. package/src/ai/pack.ts +121 -0
  46. package/src/ai/prompts.ts +87 -0
  47. package/src/ai/providers.ts +276 -0
  48. package/src/ai/queue.test.ts +97 -0
  49. package/src/ai/routes.ts +359 -0
  50. package/src/ai/store.ts +301 -0
  51. package/src/ai/wikidata.test.ts +184 -0
  52. package/src/ai/wikidata.ts +87 -0
  53. package/src/ai/worker.ts +530 -0
  54. package/src/astro/astro-modules.d.ts +6 -0
  55. package/src/astro/breadcrumbs/Breadcrumbs.astro +129 -0
  56. package/src/astro/breadcrumbs/BreadcrumbsBlock.astro +12 -0
  57. package/src/astro/codeBlocks/CodeBlock.astro +62 -0
  58. package/src/astro/codeBlocks/EmDashCode.astro +61 -0
  59. package/src/astro/customBlocks/CustomBlock.astro +64 -0
  60. package/src/astro/customBlocks/Details.astro +8 -0
  61. package/src/astro/customBlocks/Disclosure.astro +8 -0
  62. package/src/astro/customBlocks/Note.astro +8 -0
  63. package/src/astro/customBlocks/Podcast.astro +8 -0
  64. package/src/astro/customBlocks/Quote.astro +8 -0
  65. package/src/astro/customBlocks/Testimonial.astro +8 -0
  66. package/src/astro/files/FileCard.astro +103 -0
  67. package/src/astro/headings/HeadingBlock.astro +129 -0
  68. package/src/astro/headings/TableOfContents.astro +45 -0
  69. package/src/astro/headings/request.ts +19 -0
  70. package/src/astro/images/ResponsiveImage.astro +60 -0
  71. package/src/astro/index.ts +84 -0
  72. package/src/astro/reviews/CoywolfReview.astro +27 -0
  73. package/src/astro/reviews/Review.astro +49 -0
  74. package/src/astro/search/SearchBox.astro +61 -0
  75. package/src/astro/search/SearchBoxForm.astro +509 -0
  76. package/src/astro/tables/CaptionedTable.astro +72 -0
  77. package/src/astro/videos/CoywolfVideo.astro +448 -0
  78. package/src/backups/download.ts +112 -0
  79. package/src/backups/dump.test.mjs +287 -0
  80. package/src/backups/dump.ts +288 -0
  81. package/src/backups/module.ts +276 -0
  82. package/src/backups/pack.ts +33 -0
  83. package/src/backups/restore.test.mjs +103 -0
  84. package/src/backups/restore.ts +209 -0
  85. package/src/backups/store.ts +409 -0
  86. package/src/breadcrumbs/pack.ts +113 -0
  87. package/src/breadcrumbs/settings.ts +102 -0
  88. package/src/breadcrumbs/stamp.ts +40 -0
  89. package/src/breadcrumbs/trail.ts +123 -0
  90. package/src/codeBlocks/contrast.ts +179 -0
  91. package/src/codeBlocks/hljs.d.ts +5 -0
  92. package/src/codeBlocks/pack.ts +110 -0
  93. package/src/codeBlocks/render.test.ts +152 -0
  94. package/src/codeBlocks/render.ts +304 -0
  95. package/src/codeBlocks/settings.ts +18 -0
  96. package/src/codeBlocks/themes.generated.ts +304 -0
  97. package/src/codeBlocks/themes.ts +114 -0
  98. package/src/core/compose.ts +244 -0
  99. package/src/core/content-url.ts +888 -0
  100. package/src/core/csv.ts +119 -0
  101. package/src/core/d1-batch.ts +65 -0
  102. package/src/core/entry-docs.ts +175 -0
  103. package/src/core/features-module.ts +48 -0
  104. package/src/core/features.ts +392 -0
  105. package/src/core/generation.ts +88 -0
  106. package/src/core/guide-content.ts +166 -0
  107. package/src/core/module.ts +104 -0
  108. package/src/core/read-failures.ts +57 -0
  109. package/src/core/request-memo.ts +71 -0
  110. package/src/core/secrets.ts +83 -0
  111. package/src/core/settings-enhance.ts +74 -0
  112. package/src/core/site.ts +36 -0
  113. package/src/core/storage.ts +27 -0
  114. package/src/customBlocks/pack.ts +260 -0
  115. package/src/customBlocks/render.ts +424 -0
  116. package/src/customBlocks/rich.ts +313 -0
  117. package/src/customBlocks/settings.ts +46 -0
  118. package/src/discovery/build.ts +252 -0
  119. package/src/discovery/feed-build.ts +217 -0
  120. package/src/discovery/feed.ts +172 -0
  121. package/src/discovery/indexnow.ts +114 -0
  122. package/src/discovery/llms.ts +79 -0
  123. package/src/discovery/markdown.ts +294 -0
  124. package/src/discovery/memory-cache.ts +97 -0
  125. package/src/discovery/middleware.ts +186 -0
  126. package/src/discovery/module.ts +353 -0
  127. package/src/discovery/news.ts +99 -0
  128. package/src/discovery/pack.ts +58 -0
  129. package/src/discovery/settings.ts +91 -0
  130. package/src/discovery/urls.ts +35 -0
  131. package/src/env.d.ts +10 -0
  132. package/src/files/card.ts +231 -0
  133. package/src/files/counts.test.ts +114 -0
  134. package/src/files/counts.ts +60 -0
  135. package/src/files/export.ts +44 -0
  136. package/src/files/format.test.ts +93 -0
  137. package/src/files/format.ts +163 -0
  138. package/src/files/module.ts +596 -0
  139. package/src/files/pack.ts +92 -0
  140. package/src/files/r2-rules.test.ts +30 -0
  141. package/src/files/r2-rules.ts +41 -0
  142. package/src/files/range.ts +50 -0
  143. package/src/files/s3.ts +114 -0
  144. package/src/files/serve.ts +141 -0
  145. package/src/files/sigv4.test.ts +64 -0
  146. package/src/files/sigv4.ts +160 -0
  147. package/src/files/site.ts +137 -0
  148. package/src/files/walker.ts +58 -0
  149. package/src/formUploads/lib.ts +266 -0
  150. package/src/formUploads/pack.ts +55 -0
  151. package/src/formUploads/wrap.ts +592 -0
  152. package/src/forms.ts +16 -0
  153. package/src/headings/headings.test.ts +159 -0
  154. package/src/headings/pack.ts +154 -0
  155. package/src/headings/render.ts +139 -0
  156. package/src/headings/settings.ts +117 -0
  157. package/src/headings/slug.ts +68 -0
  158. package/src/headings/stamp.ts +142 -0
  159. package/src/headings/test-hooks.mjs +14 -0
  160. package/src/headings/toc.ts +48 -0
  161. package/src/images/cloudflare.ts +329 -0
  162. package/src/images/lib.ts +221 -0
  163. package/src/images/module.ts +372 -0
  164. package/src/images/pack.ts +804 -0
  165. package/src/images/pending.ts +37 -0
  166. package/src/images/settings.ts +48 -0
  167. package/src/images/setup.ts +192 -0
  168. package/src/images/variants-estimate.ts +162 -0
  169. package/src/images/variants-store.ts +475 -0
  170. package/src/images/variants.ts +311 -0
  171. package/src/index.ts +143 -0
  172. package/src/links/check.ts +101 -0
  173. package/src/links/classify.ts +270 -0
  174. package/src/links/export.ts +59 -0
  175. package/src/links/module.ts +652 -0
  176. package/src/links/pack.ts +43 -0
  177. package/src/links/pt.ts +325 -0
  178. package/src/links/store.ts +704 -0
  179. package/src/middleware.ts +287 -0
  180. package/src/modules.ts +53 -0
  181. package/src/options.ts +66 -0
  182. package/src/pageCache/lib.test.mjs +224 -0
  183. package/src/pageCache/lib.ts +274 -0
  184. package/src/pageCache/pack.ts +240 -0
  185. package/src/pageCache/usage-core.ts +152 -0
  186. package/src/pageCache/usage.ts +262 -0
  187. package/src/pageCache/warm.test.mjs +670 -0
  188. package/src/pageCache/warm.ts +722 -0
  189. package/src/redirects/middleware.ts +169 -0
  190. package/src/redirects/module.ts +124 -0
  191. package/src/redirects/pack.ts +89 -0
  192. package/src/redirects/removed-core.ts +151 -0
  193. package/src/redirects/removed.ts +293 -0
  194. package/src/redirects/rules.ts +355 -0
  195. package/src/redirects/slug-changes.ts +207 -0
  196. package/src/redirects/slug-core.test.ts +50 -0
  197. package/src/redirects/slug-core.ts +49 -0
  198. package/src/redirects/transfer.ts +92 -0
  199. package/src/reviews/lib.ts +624 -0
  200. package/src/reviews/pack.ts +135 -0
  201. package/src/reviews/schema.ts +128 -0
  202. package/src/reviews/settings.ts +26 -0
  203. package/src/robots/bots.ts +321 -0
  204. package/src/robots/data/bots.json +696 -0
  205. package/src/robots/data/verified.json +582 -0
  206. package/src/robots/directory.ts +26 -0
  207. package/src/robots/explain.ts +191 -0
  208. package/src/robots/guided.ts +300 -0
  209. package/src/robots/importer.ts +263 -0
  210. package/src/robots/middleware.ts +151 -0
  211. package/src/robots/module.ts +446 -0
  212. package/src/robots/pack.ts +69 -0
  213. package/src/robots/radar.ts +129 -0
  214. package/src/robots/rep.test.ts +335 -0
  215. package/src/robots/rep.ts +362 -0
  216. package/src/robots/rules.test.ts +251 -0
  217. package/src/robots/rules.ts +883 -0
  218. package/src/robots/transfer.ts +45 -0
  219. package/src/robots/validate.ts +524 -0
  220. package/src/schema/catalog.ts +146 -0
  221. package/src/schema/graph.test.mjs +522 -0
  222. package/src/schema/graph.ts +791 -0
  223. package/src/schema/module.ts +1138 -0
  224. package/src/schema/og-image.ts +100 -0
  225. package/src/schema/pack.ts +58 -0
  226. package/src/search/engine.ts +578 -0
  227. package/src/search/fallback.ts +80 -0
  228. package/src/search/live-cache.ts +116 -0
  229. package/src/search/live-client.ts +770 -0
  230. package/src/search/live-serve.ts +282 -0
  231. package/src/search/module.ts +261 -0
  232. package/src/search/pack.ts +57 -0
  233. package/src/search/query.ts +132 -0
  234. package/src/search/ratelimit-core.ts +118 -0
  235. package/src/search/ratelimit.ts +79 -0
  236. package/src/search/snippet.ts +135 -0
  237. package/src/search/title-index.ts +139 -0
  238. package/src/search/title-match.ts +192 -0
  239. package/src/shared.ts +71 -0
  240. package/src/version.ts +2 -0
  241. package/src/videos/date.ts +40 -0
  242. package/src/videos/engagement.ts +75 -0
  243. package/src/videos/lib.ts +679 -0
  244. package/src/videos/module.ts +1160 -0
  245. package/src/videos/pack.ts +187 -0
  246. package/src/videos/pause.ts +42 -0
  247. package/src/videos/poster.ts +324 -0
  248. package/src/videos/render.ts +336 -0
  249. package/src/videos/store.ts +551 -0
  250. package/src/videos/stream.ts +134 -0
  251. package/src/wpImport/convert.ts +702 -0
  252. package/src/wpImport/guests.ts +329 -0
  253. package/src/wpImport/gutenberg.ts +157 -0
  254. package/src/wpImport/markers.ts +66 -0
  255. package/src/wpImport/pack.ts +451 -0
  256. package/src/wpImport/parents.ts +202 -0
  257. package/src/wpImport/prepare.ts +592 -0
  258. package/src/wpImport/redirects.ts +358 -0
  259. package/src/wpImport/sources.ts +67 -0
  260. package/src/wpImport/stream.ts +160 -0
  261. package/src/wpImport/urls.ts +354 -0
@@ -0,0 +1,276 @@
1
+ /**
2
+ * Backups module: D1 dumps + R2 media mirror, with rewind (D1 Time Travel),
3
+ * undo, restore to a new database, and missing-media restore.
4
+ */
5
+ import { PluginRouteError, definePluginRoute } from "emdash";
6
+ import { z } from "zod";
7
+
8
+ import { type CronScheduler, type SettingsReader, parseInput, secret, workerEnv } from "../shared.js";
9
+ import { createDownloadLink } from "./download.js";
10
+ import { dumpLines } from "./dump.js";
11
+ import {
12
+ type RestoreConfig,
13
+ currentBookmark,
14
+ getUndo,
15
+ restoreMissingMedia,
16
+ restoreToNewDatabase,
17
+ rewind,
18
+ undoRewind,
19
+ } from "./restore.js";
20
+ import { FILE, type Manifest, STAMP, listBackups, makeStamp, mirrorBucket, mirrorMedia, pruneBackups, writeDumpStream, writeManifest } from "./store.js";
21
+
22
+ export interface BackupsOptions {
23
+ /** D1 binding of the site database. Default "DB". */
24
+ database?: string;
25
+ /** R2 binding of the media bucket. Default "MEDIA". */
26
+ media?: string;
27
+ /** R2 binding of the backup bucket (use a separate, private bucket). Default "BACKUPS". */
28
+ backups?: string;
29
+ /**
30
+ * R2 binding of the private form uploads bucket (Coywolf Pack → Private form
31
+ * uploads), mirrored to uploads/ in the backup bucket. Default "FORM_UPLOADS";
32
+ * skipped when the site has no such binding. false leaves uploads out.
33
+ */
34
+ uploads?: string | false;
35
+ /** Dump file name prefix (e.g. "mysite" -> mysite.sql.gz). Default "database". */
36
+ name?: string;
37
+ /**
38
+ * Restore via the Cloudflare API: the account and D1 database IDs, plus a
39
+ * Worker secret with an API token (Account → D1 → Edit). These live outside
40
+ * the database on purpose: a rewind rolls back plugin settings too.
41
+ */
42
+ restore?: { accountId: string; databaseId: string; tokenSecret?: string };
43
+ }
44
+
45
+ /**
46
+ * Schedule and retention, edited on the Backups page. Not in the plugin's
47
+ * settingsSchema (that holds secrets only), so every read supplies its default.
48
+ */
49
+ export const BACKUPS_DEFAULTS = { scheduled: false, retentionDays: 30, staleAfterHours: 36 } as const;
50
+
51
+ const backupsSettingsInput = z.object({
52
+ scheduled: z.boolean(),
53
+ retentionDays: z.number().int().min(1, "Keep backups for at least 1 day.").max(365, "Keep backups for at most 365 days."),
54
+ staleAfterHours: z.number().int().min(1, "Warn after at least 1 hour.").max(720, "Warn after at most 720 hours (30 days)."),
55
+ });
56
+
57
+ async function settings(ctx: SettingsReader) {
58
+ return {
59
+ scheduled: (await ctx.settings.get<boolean>("backupsScheduled")) ?? BACKUPS_DEFAULTS.scheduled,
60
+ retentionDays: (await ctx.settings.get<number>("backupsRetentionDays")) ?? BACKUPS_DEFAULTS.retentionDays,
61
+ staleAfterHours: (await ctx.settings.get<number>("backupsStaleAfterHours")) ?? BACKUPS_DEFAULTS.staleAfterHours,
62
+ };
63
+ }
64
+
65
+ /** Daily task for scheduled backups and retention. schedule() is an upsert, so it's safe to repeat. */
66
+ export const BACKUPS_TASK = "backups-daily";
67
+ export async function ensureBackupsTask(ctx: CronScheduler) {
68
+ await ctx.cron?.schedule(BACKUPS_TASK, { schedule: "@daily" });
69
+ }
70
+
71
+ const confirmed = (word: string) => z.literal(word, { message: `Type ${word} to confirm.` });
72
+
73
+ export function backupsModule(options: BackupsOptions) {
74
+ const name = options.name ?? "database";
75
+
76
+ async function bindings() {
77
+ const env = await workerEnv();
78
+ const db = env[options.database ?? "DB"] as D1Database | undefined;
79
+ const media = env[options.media ?? "MEDIA"] as R2Bucket | undefined;
80
+ const backups = env[options.backups ?? "BACKUPS"] as R2Bucket | undefined;
81
+ const uploads = options.uploads === false ? undefined : (env[options.uploads ?? "FORM_UPLOADS"] as R2Bucket | undefined);
82
+ const missing = [!db && "database", !media && "media", !backups && "backups"].filter(Boolean);
83
+ if (missing.length) throw PluginRouteError.badRequest(`Backups: missing ${missing.join(", ")} binding (see README).`);
84
+ const tokenName = options.restore?.tokenSecret ?? "BACKUPS_API_TOKEN";
85
+ const token = secret(env, tokenName);
86
+ const restore: RestoreConfig | null =
87
+ options.restore && token ? { token, accountId: options.restore.accountId, databaseId: options.restore.databaseId } : null;
88
+ const restoreStatus = restore
89
+ ? { enabled: true as const }
90
+ : {
91
+ enabled: false as const,
92
+ reason: options.restore
93
+ ? `Set the ${tokenName} Worker secret to an API token with D1 Edit permission.`
94
+ : "Add the backups.restore option (accountId, databaseId) in astro.config.mjs.",
95
+ };
96
+ return { db: db!, media: media!, backups: backups!, uploads, restore, restoreStatus };
97
+ }
98
+
99
+ async function readDump(backups: R2Bucket, stamp: string): Promise<{ manifest: Manifest; sql: string }> {
100
+ const manifestObject = await backups.get(`d1/${stamp}/manifest.json`);
101
+ if (!manifestObject) throw PluginRouteError.notFound("Backup not found");
102
+ const manifest = (await manifestObject.json()) as Manifest;
103
+ const dump = await backups.get(`d1/${stamp}/${manifest.file}`);
104
+ if (!dump) throw PluginRouteError.notFound("Backup file not found");
105
+ return { manifest, sql: await new Response(dump.body.pipeThrough(new DecompressionStream("gzip"))).text() };
106
+ }
107
+
108
+ async function runBackup(source: "admin" | "scheduled") {
109
+ const { db, media, backups, uploads, restore } = await bindings();
110
+ const stamp = makeStamp();
111
+ const timeTravelBookmark = restore ? await currentBookmark(restore).catch(() => undefined) : undefined;
112
+ const file = `${name}.sql.gz`;
113
+ // Streamed: rows are read a page at a time and gzipped straight into the bucket.
114
+ const counts = { tables: 0, rows: 0 };
115
+ const written = await writeDumpStream(backups, stamp, file, dumpLines(db, counts));
116
+ const extra = { database: name, source, timeTravelBookmark, tables: counts.tables, rows: counts.rows };
117
+ // Save the database first so a media problem can't cost the dump.
118
+ const manifest = await writeManifest(backups, stamp, file, written, extra);
119
+ let result = manifest;
120
+ try {
121
+ const mediaResult = await mirrorMedia(media, backups, stamp);
122
+ result = await writeManifest(backups, stamp, file, written, { ...extra, media: mediaResult });
123
+ } catch (error) {
124
+ console.error("coywolf backups: media mirror failed; the database backup was saved", error);
125
+ }
126
+ if (uploads) {
127
+ try {
128
+ const uploadsResult = await mirrorBucket(uploads, backups, stamp, "uploads");
129
+ result = await writeManifest(backups, stamp, file, written, { ...result, uploads: uploadsResult });
130
+ } catch (error) {
131
+ console.error("coywolf backups: form uploads mirror failed; the database backup was saved", error);
132
+ }
133
+ }
134
+ return result;
135
+ }
136
+
137
+ async function daily(ctx: SettingsReader & { log: { info(msg: string, data?: unknown): void } }) {
138
+ const s = await settings(ctx);
139
+ if (s.scheduled) {
140
+ const manifest = await runBackup("scheduled");
141
+ ctx.log.info("Scheduled backup complete", { stamp: manifest.stamp, bytes: manifest.bytes });
142
+ }
143
+ const { backups } = await bindings();
144
+ const pruned = await pruneBackups(backups, s.retentionDays);
145
+ if (pruned) ctx.log.info("Pruned old backups", { pruned });
146
+ }
147
+
148
+ const routes = {
149
+ "backups/list": {
150
+ permission: "plugins:manage" as const,
151
+ handler: async (ctx: SettingsReader & CronScheduler) => {
152
+ await ensureBackupsTask(ctx);
153
+ const { backups, restoreStatus } = await bindings();
154
+ return { items: await listBackups(backups), ...(await settings(ctx)), restore: restoreStatus, undo: await getUndo(backups) };
155
+ },
156
+ },
157
+
158
+ /** Schedule and retention (the Backups page's Settings section). */
159
+ "backups/settings/save": definePluginRoute({
160
+ permission: "plugins:manage",
161
+ methods: ["POST"],
162
+ request: { body: "json" },
163
+ handler: async (ctx) => {
164
+ const input = parseInput(backupsSettingsInput, ctx.input);
165
+ await ctx.settings.set("backupsScheduled", input.scheduled);
166
+ await ctx.settings.set("backupsRetentionDays", input.retentionDays);
167
+ await ctx.settings.set("backupsStaleAfterHours", input.staleAfterHours);
168
+ await ensureBackupsTask(ctx);
169
+ ctx.log.info("Backup settings saved", input);
170
+ return settings(ctx);
171
+ },
172
+ }),
173
+
174
+ "backups/run": definePluginRoute({
175
+ permission: "plugins:manage",
176
+ methods: ["POST"],
177
+ request: { body: "none" },
178
+ handler: async (ctx) => {
179
+ await ensureBackupsTask(ctx);
180
+ const manifest = await runBackup("admin");
181
+ ctx.log.info("Backup complete", { stamp: manifest.stamp, bytes: manifest.bytes });
182
+ const { backups } = await bindings();
183
+ await pruneBackups(backups, (await settings(ctx)).retentionDays);
184
+ return manifest;
185
+ },
186
+ }),
187
+
188
+ "backups/rewind": definePluginRoute({
189
+ permission: "plugins:manage",
190
+ methods: ["POST"],
191
+ request: { body: "json" },
192
+ handler: async (ctx) => {
193
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), confirm: confirmed("REWIND") }), ctx.input);
194
+ const { backups, restore } = await bindings();
195
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
196
+ const manifestObject = await backups.get(`d1/${input.stamp}/manifest.json`);
197
+ const manifest = manifestObject ? ((await manifestObject.json()) as Manifest) : undefined;
198
+ const undo = await rewind(restore, backups, { stamp: input.stamp, bookmark: manifest?.timeTravelBookmark });
199
+ ctx.log.warn("Database rewound", { to: input.stamp, undoBookmark: undo.bookmark });
200
+ return undo;
201
+ },
202
+ }),
203
+
204
+ "backups/undo": definePluginRoute({
205
+ permission: "plugins:manage",
206
+ methods: ["POST"],
207
+ request: { body: "json" },
208
+ handler: async (ctx) => {
209
+ parseInput(z.object({ confirm: confirmed("UNDO") }), ctx.input);
210
+ const { backups, restore } = await bindings();
211
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
212
+ const undo = await undoRewind(restore, backups);
213
+ ctx.log.warn("Rewind undone", { bookmark: undo.bookmark });
214
+ return undo;
215
+ },
216
+ }),
217
+
218
+ "backups/restore-new": definePluginRoute({
219
+ permission: "plugins:manage",
220
+ methods: ["POST"],
221
+ request: { body: "json" },
222
+ handler: async (ctx) => {
223
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), confirm: confirmed("RESTORE") }), ctx.input);
224
+ const { backups, restore } = await bindings();
225
+ if (!restore) throw PluginRouteError.badRequest("Restore isn't configured.");
226
+ const { sql } = await readDump(backups, input.stamp);
227
+ const result = await restoreToNewDatabase(restore, `${name}-restore-${input.stamp.toLowerCase()}`, sql);
228
+ ctx.log.warn("Backup restored to a new database", { stamp: input.stamp, database: result.databaseName });
229
+ return result;
230
+ },
231
+ }),
232
+
233
+ "backups/restore-media": definePluginRoute({
234
+ permission: "plugins:manage",
235
+ methods: ["POST"],
236
+ request: { body: "none" },
237
+ handler: async (ctx) => {
238
+ // Media first, then private form uploads, in batches; the admin calls again while anything is pending.
239
+ const { media, backups, uploads } = await bindings();
240
+ const result = await restoreMissingMedia(media, backups);
241
+ const fromUploads =
242
+ uploads && !result.pending ? await restoreMissingMedia(uploads, backups, 200, "uploads") : { restored: 0, checked: 0, pending: 0 };
243
+ const combined = {
244
+ restored: result.restored + fromUploads.restored,
245
+ checked: result.checked + fromUploads.checked,
246
+ // Uploads aren't looked at until media is done, so count them as pending meanwhile.
247
+ pending: result.pending + (uploads && result.pending ? 1 : fromUploads.pending),
248
+ media: result,
249
+ uploads: uploads ? fromUploads : undefined,
250
+ };
251
+ ctx.log.info("Missing media restored", combined);
252
+ return combined;
253
+ },
254
+ }),
255
+
256
+ /**
257
+ * A ten-minute link that streams one backup file (see ./download.ts).
258
+ * The file is a gzipped SQL dump that restores into any D1 database.
259
+ */
260
+ "backups/download-link": definePluginRoute({
261
+ permission: "plugins:manage",
262
+ methods: ["POST"],
263
+ request: { body: "json" },
264
+ handler: async (ctx) => {
265
+ const input = parseInput(z.object({ stamp: z.string().regex(STAMP), file: z.string().regex(FILE) }), ctx.input);
266
+ const { backups } = await bindings();
267
+ const link = await createDownloadLink(backups, input.stamp, input.file);
268
+ if (!link) throw PluginRouteError.notFound("Backup file not found");
269
+ ctx.log.info("Backup download link created", { stamp: input.stamp, file: input.file });
270
+ return link;
271
+ },
272
+ }),
273
+ };
274
+
275
+ return { routes, daily };
276
+ }
@@ -0,0 +1,33 @@
1
+ import type { PluginContext } from "emdash";
2
+
3
+ import { registerFeatures } from "../core/features.js";
4
+ import type { PackMiddleware, PackModule } from "../core/module.js";
5
+ import { configureBackupDownloads, serveBackupDownload } from "./download.js";
6
+ import { BACKUPS_TASK, type BackupsOptions, backupsModule } from "./module.js";
7
+
8
+ const FEATURES = [
9
+ { id: "backups", label: "Backups", description: "Database and media backups, rewind, and restore.", default: true },
10
+ ];
11
+ registerFeatures(FEATURES);
12
+
13
+ export function backupsPack(options: BackupsOptions): PackModule {
14
+ // Module-level so the site middleware can find the backup bucket (createPlugin() runs at isolate start).
15
+ configureBackupDownloads(options);
16
+ const backups = backupsModule(options);
17
+ return {
18
+ id: "backups",
19
+ label: "Backups",
20
+ features: FEATURES,
21
+ routes: backups.routes,
22
+ tasks: [{ name: BACKUPS_TASK, schedule: "@daily", handler: (ctx: PluginContext) => backups.daily(ctx) }],
23
+ adminPages: [{ path: "/backups", label: "Backups", icon: "database" }],
24
+ widgets: [{ id: "backup-status", title: "Backups", size: "third" }],
25
+ };
26
+ }
27
+
28
+ /** Streams backup downloads at the links the Backups page creates (see ./download.ts). */
29
+ export const backupsDownloadMiddleware: PackMiddleware = {
30
+ module: "backups",
31
+ feature: "backups",
32
+ handle: (context, env) => serveBackupDownload(context.request, context.url, env),
33
+ };
@@ -0,0 +1,103 @@
1
+ // Unit tests for restoring missing media in batches.
2
+ // Run: node --test src/backups/restore.test.mjs (Node 22.6+ strips the types from store.ts)
3
+ import assert from "node:assert/strict";
4
+ import { test } from "node:test";
5
+
6
+ const { restoreMissingMedia, mirrorBucket, pruneBackups } = await import("./store.ts");
7
+
8
+ /** The slice of an R2 bucket that restoreMissingMedia uses, paging 100 keys at a time like R2's default 1,000. */
9
+ function bucket(entries = {}) {
10
+ const store = new Map(Object.entries(entries));
11
+ return {
12
+ store,
13
+ async list({ prefix = "", cursor, delimiter } = {}) {
14
+ const keys = [...store.keys()].filter((k) => k.startsWith(prefix)).sort();
15
+ if (delimiter) {
16
+ const dirs = [...new Set(keys.map((k) => k.slice(prefix.length)).filter((k) => k.includes(delimiter)).map((k) => prefix + k.split(delimiter)[0] + delimiter))];
17
+ return { objects: [], delimitedPrefixes: dirs, truncated: false };
18
+ }
19
+ const start = cursor ? Number(cursor) : 0;
20
+ const page = keys.slice(start, start + 100);
21
+ const truncated = start + 100 < keys.length;
22
+ return { objects: page.map((key) => ({ key, etag: `e-${store.get(key)}` })), truncated, cursor: truncated ? String(start + 100) : undefined };
23
+ },
24
+ async get(key) {
25
+ return store.has(key) ? { body: store.get(key), httpMetadata: {} } : null;
26
+ },
27
+ async put(key, body) {
28
+ store.set(key, body);
29
+ },
30
+ async delete(keys) {
31
+ for (const k of [].concat(keys)) store.delete(k);
32
+ },
33
+ };
34
+ }
35
+
36
+ test("restores missing media in batches until nothing is pending, never overwriting", async () => {
37
+ const mirror = {};
38
+ for (let i = 0; i < 450; i++) mirror[`media/current/file-${String(i).padStart(3, "0")}.jpg`] = `backup-${i}`;
39
+ const backups = bucket(mirror);
40
+ const media = bucket({ "file-000.jpg": "live-0" });
41
+
42
+ const runs = [];
43
+ for (;;) {
44
+ const result = await restoreMissingMedia(media, backups, 200);
45
+ runs.push(result);
46
+ if (!result.pending) break;
47
+ }
48
+ assert.deepEqual(runs.map((r) => [r.restored, r.pending]), [[200, 249], [200, 49], [49, 0]]);
49
+ assert.equal(media.store.size, 450);
50
+ assert.equal(media.store.get("file-000.jpg"), "live-0", "existing files are left alone");
51
+ assert.equal(media.store.get("file-449.jpg"), "backup-449");
52
+ assert.deepEqual(await restoreMissingMedia(media, backups, 200), { restored: 0, checked: 450, pending: 0 });
53
+ });
54
+
55
+ test("private form uploads mirror to uploads/ and restore from there, separate from media", async () => {
56
+ const uploads = bucket({ "forms/a.pdf": "A", "forms/b.png": "B" });
57
+ const backups = bucket({ "media/current/photo.jpg": "P" });
58
+ const first = await mirrorBucket(uploads, backups, "2020-01-01T0600Z", "uploads");
59
+ assert.deepEqual(first, { copied: 2, preserved: 0, total: 2, pending: 0 });
60
+ assert.equal(backups.store.get("uploads/current/forms/a.pdf"), "A");
61
+
62
+ // A replaced upload keeps its old copy under uploads/changed/<stamp>/; a deleted one too.
63
+ uploads.store.set("forms/a.pdf", "A2");
64
+ uploads.store.delete("forms/b.png");
65
+ await mirrorBucket(uploads, backups, "2020-01-02T0600Z", "uploads");
66
+ assert.equal(backups.store.get("uploads/current/forms/a.pdf"), "A2");
67
+ assert.equal(backups.store.get("uploads/changed/2020-01-02T0600Z/forms/a.pdf"), "A");
68
+ assert.equal(backups.store.get("uploads/changed/2020-01-02T0600Z/forms/b.png"), "B");
69
+ assert.ok(!backups.store.has("uploads/current/forms/b.png"));
70
+ assert.equal(backups.store.get("media/current/photo.jpg"), "P", "media mirror untouched");
71
+
72
+ const empty = bucket();
73
+ const restored = await restoreMissingMedia(empty, backups, 200, "uploads");
74
+ assert.deepEqual(restored, { restored: 1, checked: 1, pending: 0 });
75
+ assert.equal(empty.store.get("forms/a.pdf"), "A2");
76
+ assert.ok(!empty.store.has("photo.jpg"), "media isn't restored into the uploads bucket");
77
+
78
+ // Old replaced copies are pruned like media's.
79
+ assert.equal(await pruneBackups(backups, 1), 0);
80
+ assert.ok(![...backups.store.keys()].some((k) => k.startsWith("uploads/changed/2020-01-02")), "uploads/changed pruned");
81
+ });
82
+
83
+ test("the mirror copies a few files at a time, never more than its limit", async () => {
84
+ const { MIRROR_CONCURRENCY } = await import("./store.ts");
85
+ const files = {};
86
+ for (let i = 0; i < 40; i++) files[`f${i}.jpg`] = `v${i}`;
87
+ const media = bucket(files);
88
+ const backups = bucket({ "media/current/f0.jpg": "old", "media/current/gone.jpg": "G" });
89
+ let inFlight = 0;
90
+ let most = 0;
91
+ const slowGet = media.get.bind(media);
92
+ media.get = async (key) => {
93
+ most = Math.max(most, ++inFlight);
94
+ await new Promise((resolve) => setTimeout(resolve, 2));
95
+ inFlight--;
96
+ return slowGet(key);
97
+ };
98
+ const result = await mirrorBucket(media, backups, "2020-01-03T0600Z", "media", 30);
99
+ assert.equal(most, MIRROR_CONCURRENCY);
100
+ assert.deepEqual(result, { copied: 30, preserved: 1, total: 40, pending: 11 });
101
+ assert.equal(backups.store.get("media/changed/2020-01-03T0600Z/f0.jpg"), "old", "a replaced file's old copy is kept before it's overwritten");
102
+ assert.equal(backups.store.get("media/current/f0.jpg"), "v0");
103
+ });
@@ -0,0 +1,209 @@
1
+ /**
2
+ * Restores, through the Cloudflare API (a Worker can't create databases or
3
+ * time-travel through its D1 binding).
4
+ *
5
+ * Rewinding rolls the live database back, including this plugin's own
6
+ * settings and storage, so everything a restore depends on lives outside the
7
+ * database: the API token is a Worker secret, the account and database IDs are
8
+ * plugin options, and undo points are written to the backup bucket.
9
+ */
10
+ import { PluginRouteError } from "emdash";
11
+
12
+ import { stampToDate } from "./store.js";
13
+
14
+ const API = "https://api.cloudflare.com/client/v4";
15
+ /** One file per rewind, newest last: undo pops the most recent, so earlier states stay reachable. */
16
+ const UNDO_PREFIX = "restore/undo/";
17
+
18
+ export interface RestoreConfig {
19
+ token: string;
20
+ accountId: string;
21
+ databaseId: string;
22
+ }
23
+
24
+ export interface UndoPoint {
25
+ /** Bookmark of the database just before the rewind. */
26
+ bookmark: string;
27
+ rewoundAt: string;
28
+ rewoundTo: string;
29
+ }
30
+
31
+ async function cf<T>(config: RestoreConfig, path: string, init: RequestInit = {}): Promise<T> {
32
+ const res = await fetch(`${API}/accounts/${config.accountId}${path}`, {
33
+ ...init,
34
+ headers: { Authorization: `Bearer ${config.token}`, "Content-Type": "application/json", ...init.headers },
35
+ });
36
+ const body = (await res.json().catch(() => ({}))) as { success?: boolean; result?: T; errors?: Array<{ message: string }> };
37
+ if (!res.ok || body.success === false) {
38
+ // Surface the API's message in the admin (plain Errors become a generic 500).
39
+ throw PluginRouteError.badRequest(
40
+ `Cloudflare API ${res.status}: ${body.errors?.map((e) => e.message).join("; ") || res.statusText}`,
41
+ );
42
+ }
43
+ return body.result as T;
44
+ }
45
+
46
+ const db = (config: RestoreConfig) => `/d1/database/${config.databaseId}`;
47
+
48
+ export async function currentBookmark(config: RestoreConfig): Promise<string> {
49
+ const result = await cf<{ bookmark: string }>(config, `${db(config)}/time_travel/bookmark`);
50
+ return result.bookmark;
51
+ }
52
+
53
+ /** Roll the live database back to a backup's moment, saving an undo point first. */
54
+ export async function rewind(
55
+ config: RestoreConfig,
56
+ backups: R2Bucket,
57
+ target: { stamp: string; bookmark?: string },
58
+ ): Promise<UndoPoint> {
59
+ const query = target.bookmark
60
+ ? `bookmark=${encodeURIComponent(target.bookmark)}`
61
+ : `timestamp=${encodeURIComponent(stampToDate(target.stamp).toISOString())}`;
62
+ const before = await currentBookmark(config);
63
+ const undo: UndoPoint = { bookmark: before, rewoundAt: new Date().toISOString(), rewoundTo: target.stamp };
64
+ await backups.put(`${UNDO_PREFIX}${undo.rewoundAt}.json`, JSON.stringify(undo), {
65
+ httpMetadata: { contentType: "application/json" },
66
+ });
67
+ await cf(config, `${db(config)}/time_travel/restore?${query}`, { method: "POST" });
68
+ return undo;
69
+ }
70
+
71
+ async function latestUndoKey(backups: R2Bucket): Promise<string | null> {
72
+ const keys: string[] = [];
73
+ let cursor: string | undefined;
74
+ do {
75
+ const page = await backups.list({ prefix: UNDO_PREFIX, cursor });
76
+ for (const o of page.objects) keys.push(o.key);
77
+ cursor = page.truncated ? page.cursor : undefined;
78
+ } while (cursor);
79
+ return keys.sort().at(-1) ?? null;
80
+ }
81
+
82
+ /** The most recent rewind that hasn't been undone. */
83
+ export async function getUndo(backups: R2Bucket): Promise<UndoPoint | null> {
84
+ const key = await latestUndoKey(backups);
85
+ const object = key ? await backups.get(key) : null;
86
+ return object ? ((await object.json()) as UndoPoint) : null;
87
+ }
88
+
89
+ /** Return the database to where it was before the most recent rewind. */
90
+ export async function undoRewind(config: RestoreConfig, backups: R2Bucket): Promise<UndoPoint> {
91
+ const key = await latestUndoKey(backups);
92
+ const object = key ? await backups.get(key) : null;
93
+ if (!key || !object) throw PluginRouteError.badRequest("There is no rewind to undo.");
94
+ const undo = (await object.json()) as UndoPoint;
95
+ await cf(config, `${db(config)}/time_travel/restore?bookmark=${encodeURIComponent(undo.bookmark)}`, { method: "POST" });
96
+ await backups.delete(key);
97
+ return undo;
98
+ }
99
+
100
+ async function md5(bytes: Uint8Array<ArrayBuffer>): Promise<string> {
101
+ // MD5 is what D1's import API uses as the upload ETag (Workers support it in WebCrypto).
102
+ const digest = await crypto.subtle.digest("MD5", bytes);
103
+ return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
104
+ }
105
+
106
+ interface ImportStatus {
107
+ status?: "active" | "complete" | "error";
108
+ at_bookmark?: string;
109
+ upload_url?: string;
110
+ filename?: string;
111
+ error?: string;
112
+ success?: boolean;
113
+ }
114
+
115
+ export interface NewDatabaseRestore {
116
+ databaseId: string;
117
+ databaseName: string;
118
+ tables: number;
119
+ rows: number;
120
+ mismatches: Record<string, [number, number | undefined]>;
121
+ }
122
+
123
+ /**
124
+ * Import a dump into a brand-new D1 database and verify row counts. The live
125
+ * site keeps running on its current database until its binding is switched.
126
+ */
127
+ export async function restoreToNewDatabase(
128
+ config: RestoreConfig,
129
+ databaseName: string,
130
+ sql: string,
131
+ ): Promise<NewDatabaseRestore> {
132
+ const suffix = crypto.randomUUID().slice(0, 6);
133
+ const created = await cf<{ uuid: string; name: string }>(config, "/d1/database", {
134
+ method: "POST",
135
+ body: JSON.stringify({ name: `${databaseName}-${suffix}` }),
136
+ });
137
+ const target: RestoreConfig = { ...config, databaseId: created.uuid };
138
+ try {
139
+ return await importAndVerify(target, created, sql);
140
+ } catch (error) {
141
+ // Don't leave a half-imported database behind.
142
+ await cf(config, `/d1/database/${created.uuid}`, { method: "DELETE" }).catch(() => undefined);
143
+ throw error;
144
+ }
145
+ }
146
+
147
+ async function importAndVerify(
148
+ target: RestoreConfig,
149
+ created: { uuid: string; name: string },
150
+ sql: string,
151
+ ): Promise<NewDatabaseRestore> {
152
+ const path = `${db(target)}/import`;
153
+
154
+ const bytes = new TextEncoder().encode(sql) as Uint8Array<ArrayBuffer>;
155
+ const etag = await md5(bytes);
156
+ const init = await cf<ImportStatus>(target, path, { method: "POST", body: JSON.stringify({ action: "init", etag }) });
157
+ if (init.upload_url) {
158
+ const put = await fetch(init.upload_url, { method: "PUT", body: bytes });
159
+ if (!put.ok) throw PluginRouteError.badRequest(`Uploading the dump failed (${put.status})`);
160
+ }
161
+ let status = await cf<ImportStatus>(target, path, {
162
+ method: "POST",
163
+ body: JSON.stringify({ action: "ingest", etag, filename: init.filename }),
164
+ });
165
+ const bookmark = status.at_bookmark;
166
+ for (let i = 0; status.status !== "complete" && status.status !== "error" && i < 60; i++) {
167
+ await new Promise((r) => setTimeout(r, 1000));
168
+ status = await cf<ImportStatus>(target, path, {
169
+ method: "POST",
170
+ body: JSON.stringify({ action: "poll", current_bookmark: bookmark }),
171
+ });
172
+ }
173
+ if (status.status !== "complete") {
174
+ throw PluginRouteError.badRequest(`Import ${status.status ?? "timed out"}: ${status.error ?? "no details"}`);
175
+ }
176
+
177
+ // Verify: rows per table in the dump vs. the new database.
178
+ const expected = new Map<string, number>();
179
+ for (const match of sql.matchAll(/^INSERT INTO "((?:[^"]|"")+)"/gm)) {
180
+ const table = match[1].replace(/""/g, '"');
181
+ expected.set(table, (expected.get(table) ?? 0) + 1);
182
+ }
183
+ const tables = [...expected.keys()];
184
+ const actual = new Map<string, number>();
185
+ for (let i = 0; i < tables.length; i += 5) {
186
+ // D1 allows at most 5 terms in a compound SELECT.
187
+ const sqlCounts = tables
188
+ .slice(i, i + 5)
189
+ .map((t) => `SELECT '${t.replace(/'/g, "''")}' AS t, count(*) AS n FROM "${t.replace(/"/g, '""')}"`)
190
+ .join(" UNION ALL ");
191
+ const result = await cf<Array<{ results: Array<{ t: string; n: number }> }>>(target, `${db(target)}/query`, {
192
+ method: "POST",
193
+ body: JSON.stringify({ sql: sqlCounts }),
194
+ });
195
+ for (const row of result[0]?.results ?? []) actual.set(row.t, row.n);
196
+ }
197
+ const mismatches: NewDatabaseRestore["mismatches"] = {};
198
+ for (const [table, n] of expected) if (actual.get(table) !== n) mismatches[table] = [n, actual.get(table)];
199
+
200
+ return {
201
+ databaseId: created.uuid,
202
+ databaseName: created.name,
203
+ tables: tables.length,
204
+ rows: [...expected.values()].reduce((a, b) => a + b, 0),
205
+ mismatches,
206
+ };
207
+ }
208
+
209
+ export { restoreMissingMedia } from "./store.js";