@coywolf/emdash 0.0.0-stage → 0.35.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +203 -2
  3. package/docs/README.md +45 -0
  4. package/docs/ai.md +39 -0
  5. package/docs/backups.md +88 -0
  6. package/docs/breadcrumbs.md +59 -0
  7. package/docs/code-blocks.md +21 -0
  8. package/docs/content-urls.md +41 -0
  9. package/docs/custom-blocks.md +27 -0
  10. package/docs/discovery.md +32 -0
  11. package/docs/files.md +44 -0
  12. package/docs/form-uploads.md +59 -0
  13. package/docs/headings.md +37 -0
  14. package/docs/images.md +141 -0
  15. package/docs/internals.md +13 -0
  16. package/docs/links.md +34 -0
  17. package/docs/performance.md +72 -0
  18. package/docs/redirects.md +84 -0
  19. package/docs/releasing.md +21 -0
  20. package/docs/reviews.md +89 -0
  21. package/docs/robots.md +81 -0
  22. package/docs/schema.md +79 -0
  23. package/docs/search.md +114 -0
  24. package/docs/videos.md +48 -0
  25. package/docs/wordpress-import.md +198 -0
  26. package/package.json +79 -5
  27. package/src/admin/admin-css.ts +57 -0
  28. package/src/admin/ai.tsx +915 -0
  29. package/src/admin/backups.tsx +613 -0
  30. package/src/admin/breadcrumbs.tsx +542 -0
  31. package/src/admin/codeBlocks.tsx +204 -0
  32. package/src/admin/customBlocks.tsx +321 -0
  33. package/src/admin/dirty.ts +20 -0
  34. package/src/admin/discovery.tsx +463 -0
  35. package/src/admin/download.ts +28 -0
  36. package/src/admin/feature-switches.tsx +124 -0
  37. package/src/admin/features.tsx +142 -0
  38. package/src/admin/files.tsx +791 -0
  39. package/src/admin/formUploads.tsx +445 -0
  40. package/src/admin/guides.tsx +18 -0
  41. package/src/admin/headings.tsx +262 -0
  42. package/src/admin/images.tsx +825 -0
  43. package/src/admin/index.tsx +62 -0
  44. package/src/admin/links.tsx +980 -0
  45. package/src/admin/performance.tsx +330 -0
  46. package/src/admin/preview.tsx +63 -0
  47. package/src/admin/redirects-removed.tsx +266 -0
  48. package/src/admin/redirects.tsx +470 -0
  49. package/src/admin/reviews.tsx +232 -0
  50. package/src/admin/robots-bots.tsx +647 -0
  51. package/src/admin/robots-shared.tsx +109 -0
  52. package/src/admin/robots-wizard.tsx +748 -0
  53. package/src/admin/robots.tsx +911 -0
  54. package/src/admin/save-bar.tsx +134 -0
  55. package/src/admin/schema.tsx +1187 -0
  56. package/src/admin/search.tsx +355 -0
  57. package/src/admin/settings-ui.tsx +87 -0
  58. package/src/admin/trash-prompt-core.ts +135 -0
  59. package/src/admin/trash-prompt.tsx +156 -0
  60. package/src/admin/usage.tsx +301 -0
  61. package/src/admin/videos-display.tsx +248 -0
  62. package/src/admin/videos.tsx +1202 -0
  63. package/src/admin/wpImport.tsx +1516 -0
  64. package/src/ai/entities.ts +59 -0
  65. package/src/ai/export.ts +61 -0
  66. package/src/ai/logic.ts +606 -0
  67. package/src/ai/pack.ts +121 -0
  68. package/src/ai/prompts.ts +87 -0
  69. package/src/ai/providers.ts +276 -0
  70. package/src/ai/routes.ts +359 -0
  71. package/src/ai/store.ts +301 -0
  72. package/src/ai/wikidata.ts +87 -0
  73. package/src/ai/worker.ts +530 -0
  74. package/src/astro/astro-modules.d.ts +6 -0
  75. package/src/astro/breadcrumbs/Breadcrumbs.astro +129 -0
  76. package/src/astro/breadcrumbs/BreadcrumbsBlock.astro +12 -0
  77. package/src/astro/codeBlocks/CodeBlock.astro +62 -0
  78. package/src/astro/codeBlocks/EmDashCode.astro +61 -0
  79. package/src/astro/customBlocks/CustomBlock.astro +64 -0
  80. package/src/astro/customBlocks/Details.astro +8 -0
  81. package/src/astro/customBlocks/Disclosure.astro +8 -0
  82. package/src/astro/customBlocks/Note.astro +8 -0
  83. package/src/astro/customBlocks/Podcast.astro +8 -0
  84. package/src/astro/customBlocks/Quote.astro +8 -0
  85. package/src/astro/customBlocks/Testimonial.astro +8 -0
  86. package/src/astro/files/FileCard.astro +103 -0
  87. package/src/astro/headings/HeadingBlock.astro +129 -0
  88. package/src/astro/headings/TableOfContents.astro +45 -0
  89. package/src/astro/headings/request.ts +19 -0
  90. package/src/astro/images/ResponsiveImage.astro +60 -0
  91. package/src/astro/index.ts +84 -0
  92. package/src/astro/reviews/CoywolfReview.astro +27 -0
  93. package/src/astro/reviews/Review.astro +49 -0
  94. package/src/astro/search/SearchBox.astro +61 -0
  95. package/src/astro/search/SearchBoxForm.astro +509 -0
  96. package/src/astro/tables/CaptionedTable.astro +72 -0
  97. package/src/astro/videos/CoywolfVideo.astro +448 -0
  98. package/src/backups/download.ts +112 -0
  99. package/src/backups/dump.ts +288 -0
  100. package/src/backups/module.ts +276 -0
  101. package/src/backups/pack.ts +33 -0
  102. package/src/backups/restore.ts +209 -0
  103. package/src/backups/store.ts +409 -0
  104. package/src/breadcrumbs/pack.ts +113 -0
  105. package/src/breadcrumbs/settings.ts +102 -0
  106. package/src/breadcrumbs/stamp.ts +40 -0
  107. package/src/breadcrumbs/trail.ts +123 -0
  108. package/src/codeBlocks/contrast.ts +179 -0
  109. package/src/codeBlocks/hljs.d.ts +5 -0
  110. package/src/codeBlocks/pack.ts +110 -0
  111. package/src/codeBlocks/render.ts +304 -0
  112. package/src/codeBlocks/settings.ts +18 -0
  113. package/src/codeBlocks/themes.generated.ts +304 -0
  114. package/src/codeBlocks/themes.ts +114 -0
  115. package/src/core/compose.ts +244 -0
  116. package/src/core/content-url.ts +888 -0
  117. package/src/core/csv.ts +119 -0
  118. package/src/core/d1-batch.ts +65 -0
  119. package/src/core/entry-docs.ts +175 -0
  120. package/src/core/features-module.ts +48 -0
  121. package/src/core/features.ts +392 -0
  122. package/src/core/generation.ts +88 -0
  123. package/src/core/guide-content.ts +166 -0
  124. package/src/core/module.ts +104 -0
  125. package/src/core/read-failures.ts +57 -0
  126. package/src/core/request-memo.ts +71 -0
  127. package/src/core/secrets.ts +83 -0
  128. package/src/core/settings-enhance.ts +74 -0
  129. package/src/core/site.ts +36 -0
  130. package/src/core/storage.ts +27 -0
  131. package/src/customBlocks/pack.ts +260 -0
  132. package/src/customBlocks/render.ts +424 -0
  133. package/src/customBlocks/rich.ts +313 -0
  134. package/src/customBlocks/settings.ts +46 -0
  135. package/src/discovery/build.ts +252 -0
  136. package/src/discovery/feed-build.ts +217 -0
  137. package/src/discovery/feed.ts +172 -0
  138. package/src/discovery/indexnow.ts +114 -0
  139. package/src/discovery/llms.ts +79 -0
  140. package/src/discovery/markdown.ts +294 -0
  141. package/src/discovery/memory-cache.ts +97 -0
  142. package/src/discovery/middleware.ts +186 -0
  143. package/src/discovery/module.ts +353 -0
  144. package/src/discovery/news.ts +99 -0
  145. package/src/discovery/pack.ts +58 -0
  146. package/src/discovery/settings.ts +91 -0
  147. package/src/discovery/urls.ts +35 -0
  148. package/src/env.d.ts +10 -0
  149. package/src/files/card.ts +231 -0
  150. package/src/files/counts.ts +60 -0
  151. package/src/files/export.ts +44 -0
  152. package/src/files/format.ts +163 -0
  153. package/src/files/module.ts +596 -0
  154. package/src/files/pack.ts +92 -0
  155. package/src/files/r2-rules.ts +41 -0
  156. package/src/files/range.ts +50 -0
  157. package/src/files/s3.ts +114 -0
  158. package/src/files/serve.ts +141 -0
  159. package/src/files/sigv4.ts +160 -0
  160. package/src/files/site.ts +137 -0
  161. package/src/files/walker.ts +58 -0
  162. package/src/formUploads/lib.ts +266 -0
  163. package/src/formUploads/pack.ts +55 -0
  164. package/src/formUploads/wrap.ts +592 -0
  165. package/src/forms.ts +16 -0
  166. package/src/headings/pack.ts +154 -0
  167. package/src/headings/render.ts +139 -0
  168. package/src/headings/settings.ts +117 -0
  169. package/src/headings/slug.ts +68 -0
  170. package/src/headings/stamp.ts +142 -0
  171. package/src/headings/test-hooks.mjs +14 -0
  172. package/src/headings/toc.ts +48 -0
  173. package/src/images/cloudflare.ts +329 -0
  174. package/src/images/lib.ts +221 -0
  175. package/src/images/module.ts +372 -0
  176. package/src/images/pack.ts +804 -0
  177. package/src/images/pending.ts +37 -0
  178. package/src/images/settings.ts +48 -0
  179. package/src/images/setup.ts +192 -0
  180. package/src/images/variants-estimate.ts +162 -0
  181. package/src/images/variants-store.ts +475 -0
  182. package/src/images/variants.ts +311 -0
  183. package/src/index.ts +143 -0
  184. package/src/links/check.ts +101 -0
  185. package/src/links/classify.ts +270 -0
  186. package/src/links/export.ts +59 -0
  187. package/src/links/module.ts +652 -0
  188. package/src/links/pack.ts +43 -0
  189. package/src/links/pt.ts +325 -0
  190. package/src/links/store.ts +704 -0
  191. package/src/middleware.ts +287 -0
  192. package/src/modules.ts +53 -0
  193. package/src/options.ts +66 -0
  194. package/src/pageCache/lib.ts +274 -0
  195. package/src/pageCache/pack.ts +240 -0
  196. package/src/pageCache/usage-core.ts +152 -0
  197. package/src/pageCache/usage.ts +262 -0
  198. package/src/pageCache/warm.ts +722 -0
  199. package/src/redirects/middleware.ts +169 -0
  200. package/src/redirects/module.ts +124 -0
  201. package/src/redirects/pack.ts +89 -0
  202. package/src/redirects/removed-core.ts +151 -0
  203. package/src/redirects/removed.ts +293 -0
  204. package/src/redirects/rules.ts +355 -0
  205. package/src/redirects/slug-changes.ts +207 -0
  206. package/src/redirects/slug-core.ts +49 -0
  207. package/src/redirects/transfer.ts +92 -0
  208. package/src/reviews/lib.ts +624 -0
  209. package/src/reviews/pack.ts +135 -0
  210. package/src/reviews/schema.ts +128 -0
  211. package/src/reviews/settings.ts +26 -0
  212. package/src/robots/bots.ts +321 -0
  213. package/src/robots/data/bots.json +696 -0
  214. package/src/robots/data/verified.json +582 -0
  215. package/src/robots/directory.ts +26 -0
  216. package/src/robots/explain.ts +191 -0
  217. package/src/robots/guided.ts +300 -0
  218. package/src/robots/importer.ts +263 -0
  219. package/src/robots/middleware.ts +151 -0
  220. package/src/robots/module.ts +446 -0
  221. package/src/robots/pack.ts +69 -0
  222. package/src/robots/radar.ts +129 -0
  223. package/src/robots/rep.ts +362 -0
  224. package/src/robots/rules.ts +883 -0
  225. package/src/robots/transfer.ts +45 -0
  226. package/src/robots/validate.ts +524 -0
  227. package/src/schema/catalog.ts +146 -0
  228. package/src/schema/graph.ts +791 -0
  229. package/src/schema/module.ts +1138 -0
  230. package/src/schema/og-image.ts +100 -0
  231. package/src/schema/pack.ts +58 -0
  232. package/src/search/engine.ts +578 -0
  233. package/src/search/fallback.ts +80 -0
  234. package/src/search/live-cache.ts +116 -0
  235. package/src/search/live-client.ts +770 -0
  236. package/src/search/live-serve.ts +282 -0
  237. package/src/search/module.ts +261 -0
  238. package/src/search/pack.ts +57 -0
  239. package/src/search/query.ts +132 -0
  240. package/src/search/ratelimit-core.ts +118 -0
  241. package/src/search/ratelimit.ts +79 -0
  242. package/src/search/snippet.ts +135 -0
  243. package/src/search/title-index.ts +139 -0
  244. package/src/search/title-match.ts +192 -0
  245. package/src/shared.ts +71 -0
  246. package/src/version.ts +2 -0
  247. package/src/videos/date.ts +40 -0
  248. package/src/videos/engagement.ts +75 -0
  249. package/src/videos/lib.ts +679 -0
  250. package/src/videos/module.ts +1160 -0
  251. package/src/videos/pack.ts +187 -0
  252. package/src/videos/pause.ts +42 -0
  253. package/src/videos/poster.ts +324 -0
  254. package/src/videos/render.ts +336 -0
  255. package/src/videos/store.ts +551 -0
  256. package/src/videos/stream.ts +134 -0
  257. package/src/wpImport/convert.ts +702 -0
  258. package/src/wpImport/guests.ts +329 -0
  259. package/src/wpImport/gutenberg.ts +157 -0
  260. package/src/wpImport/markers.ts +66 -0
  261. package/src/wpImport/pack.ts +451 -0
  262. package/src/wpImport/parents.ts +202 -0
  263. package/src/wpImport/prepare.ts +592 -0
  264. package/src/wpImport/redirects.ts +358 -0
  265. package/src/wpImport/sources.ts +67 -0
  266. package/src/wpImport/stream.ts +160 -0
  267. package/src/wpImport/urls.ts +354 -0
@@ -0,0 +1,451 @@
1
+ /**
2
+ * WordPress import module: finishes a WordPress → EmDash move. EmDash's own
3
+ * importer has no plugin hook, so this works around what it drops or misses.
4
+ *
5
+ * - While on, every save (including each entry EmDash's WordPress importer
6
+ * creates) runs the converter (./convert.ts), so a prepared export
7
+ * (./prepare.ts) imports straight into native blocks: heading ids, core
8
+ * Details, and blocks from Coywolf's WordPress plugins. It runs before
9
+ * Headings & TOC, so imported heading ids become the headings' anchors.
10
+ * - "Convert imported content" scans every entry (dry run first) and converts
11
+ * what's already in the database, draft-aware, never overwriting an entry
12
+ * that changed while it ran. Given a URL map (./urls.ts), the same scan
13
+ * rewrites old `/wp-content/` URLs to media library files.
14
+ * - "Find old-site URLs" lists every `/wp-content/` URL of the old site left
15
+ * in content, for the WordPress import page to map, import and redirect.
16
+ * - Video facts from WordPress (name, length, upload date, size) fill the
17
+ * Videos module's per-video data where it has none, so players size
18
+ * correctly and VideoObject schema has a duration without a Stream token.
19
+ * - Coywolf plugins: Coywolf Files records and Video Manager library data
20
+ * (descriptions, posters, MP4 links) can be imported from WordPress's
21
+ * database.
22
+ */
23
+ import type { PluginContext, StorageCollection } from "emdash";
24
+ import { PluginRouteError, definePluginRoute } from "emdash";
25
+ import { z } from "zod";
26
+
27
+ import { ctxFeatures, isOn, registerFeatures, requireFeature } from "../core/features.js";
28
+ import type { PackModule } from "../core/module.js";
29
+ import { COLLECTIONS as FILE_COLLECTIONS, type UploadDoc, invalidateSiteCache } from "../files/site.js";
30
+ import { WORDPRESS_FILE_ID } from "../files/format.js";
31
+ import { latestData } from "../links/store.js";
32
+ import { parseInput } from "../shared.js";
33
+ import { type VideoMeta, invalidatePublicConfig, metaStore, SETTINGS as VIDEO_SETTINGS } from "../videos/store.js";
34
+ import { type ConvertOptions, type VideoFact, convertEntryData, mightNeedConversion } from "./convert.js";
35
+ import { type ImportDefaults, defaultsFromWordPress, parseFileRows } from "./sources.js";
36
+ import { findSiteUrls, rewriteSiteUrls, siteUrlPattern } from "./urls.js";
37
+
38
+ export const F = { main: "wpImport" } as const;
39
+
40
+ const FEATURES = [
41
+ {
42
+ id: F.main,
43
+ label: "WordPress import",
44
+ description:
45
+ "Tools to finish a move from WordPress: keep heading ids, reusable blocks and Details blocks, credit co-authors and guest authors, restore category and page parents, point old /wp-content/ URLs at the media library, and bring over redirects. Blocks from Coywolf's WordPress plugins become Coywolf Pack blocks as content is imported or saved.",
46
+ default: false,
47
+ },
48
+ ];
49
+ registerFeatures(FEATURES);
50
+
51
+ export const DEFAULTS_SETTING = "wpImportDefaults";
52
+
53
+ function normalizeDefaults(value: unknown): ImportDefaults {
54
+ if (!value || typeof value !== "object") return {};
55
+ const v = value as ImportDefaults;
56
+ return { video: v.video && typeof v.video === "object" ? v.video : undefined, files: v.files && typeof v.files === "object" ? v.files : undefined };
57
+ }
58
+
59
+ async function convertOptions(ctx: PluginContext): Promise<ConvertOptions> {
60
+ const d = normalizeDefaults(await ctx.settings.get(DEFAULTS_SETTING));
61
+ const features = await ctxFeatures(ctx);
62
+ // Custom Blocks that are off aren't converted to: their markers stay HTML until the block is turned on.
63
+ const customBlocks = {
64
+ note: isOn(features, "customBlocks.note"),
65
+ details: isOn(features, "customBlocks.details"),
66
+ disclosure: isOn(features, "customBlocks.disclosure"),
67
+ quote: isOn(features, "customBlocks.quote"),
68
+ testimonial: isOn(features, "customBlocks.testimonial"),
69
+ podcast: isOn(features, "customBlocks.podcast"),
70
+ };
71
+ return { videoDefaults: d.video, fileDefaults: d.files, customBlocks };
72
+ }
73
+
74
+ // ── Video facts ──────────────────────────────────────────────────
75
+
76
+ /**
77
+ * Fill in the Videos module's per-video data from WordPress where it has
78
+ * none. Stream's own data (library refresh, webhook) replaces it later.
79
+ * Returns how many videos were added.
80
+ */
81
+ async function seedVideos(ctx: PluginContext, facts: VideoFact[]): Promise<number> {
82
+ const store = (ctx.storage as Record<string, StorageCollection<VideoMeta> | undefined>).videosMeta ? metaStore(ctx) : null;
83
+ if (!store || !facts.length) return 0;
84
+ const byUid = new Map<string, VideoFact>();
85
+ for (const f of facts) byUid.set(f.uid, { ...byUid.get(f.uid), ...Object.fromEntries(Object.entries(f).filter(([, v]) => v !== undefined)) } as VideoFact);
86
+ const uids = [...byUid.keys()].slice(0, 50);
87
+ const existing = await store.getMany(uids);
88
+ const add: Array<{ id: string; data: VideoMeta }> = [];
89
+ let host: string | undefined;
90
+ for (const uid of uids) {
91
+ const fact = byUid.get(uid) as VideoFact;
92
+ host ??= fact.host;
93
+ if (existing.has(uid)) continue;
94
+ const doc: VideoMeta = { uid, updatedAt: new Date().toISOString() };
95
+ if (fact.name) doc.name = fact.name;
96
+ if (fact.duration) doc.duration = fact.duration;
97
+ if (fact.created) doc.created = fact.created;
98
+ if (fact.width && fact.height) {
99
+ doc.width = fact.width;
100
+ doc.height = fact.height;
101
+ }
102
+ add.push({ id: uid, data: doc });
103
+ }
104
+ if (add.length) await store.putMany(add);
105
+ // The customer subdomain, when the Videos page hasn't learned one yet.
106
+ if (host && !(await ctx.settings.get<string>(VIDEO_SETTINGS.host))) {
107
+ await ctx.settings.set(VIDEO_SETTINGS.host, host);
108
+ invalidatePublicConfig();
109
+ }
110
+ return add.length;
111
+ }
112
+
113
+ // ── Save hook ────────────────────────────────────────────────────
114
+
115
+ async function beforeSave(event: { content: Record<string, unknown>; collection: string }, ctx: PluginContext) {
116
+ if (!event.content || !mightNeedConversion(event.content)) return undefined;
117
+ const result = convertEntryData(event.content, await convertOptions(ctx));
118
+ if (result.videos.length) {
119
+ try {
120
+ await seedVideos(ctx, result.videos);
121
+ } catch (error) {
122
+ ctx.log.warn("WordPress import: could not record video details", { error: String(error) });
123
+ }
124
+ }
125
+ if (!result.changed) return undefined;
126
+ ctx.log.info("WordPress import: converted blocks", { collection: event.collection, changes: result.changes.length });
127
+ return result.value;
128
+ }
129
+
130
+ // ── Converting stored content ────────────────────────────────────
131
+
132
+ interface ScanState {
133
+ collections: string[];
134
+ index: number;
135
+ cursor: string | null;
136
+ }
137
+
138
+ export interface ScanEntry {
139
+ collection: string;
140
+ id: string;
141
+ title: string;
142
+ status: string;
143
+ /** "coywolf-video" → 2, "anchor" → 14, … */
144
+ changes: Record<string, number>;
145
+ result?: "updated" | "published" | "draft" | "conflict" | "failed";
146
+ error?: string;
147
+ }
148
+
149
+ const stateInput = z.object({ collections: z.array(z.string().max(100)).max(200), index: z.number().int().min(0), cursor: z.string().max(2000).nullable() }).nullish();
150
+
151
+ const scanInput = z.object({
152
+ apply: z.boolean(),
153
+ state: stateInput,
154
+ /** Old-site URL → media library URL, to rewrite in content (see ./urls.ts). */
155
+ urlMap: z.record(z.string().max(2000), z.string().max(2000)).optional(),
156
+ });
157
+
158
+ const findInput = z.object({
159
+ hosts: z.array(z.string().max(253)).max(50),
160
+ prefixes: z.array(z.string().max(200).regex(/^(?:\/[\w.~-]+){1,3}$/)).max(5).optional(),
161
+ state: stateInput,
162
+ });
163
+
164
+ /** Rewrite mapped URLs in an entry's Portable Text and text fields (media fields are left to EmDash). */
165
+ function rewriteEntryUrls(data: Record<string, unknown>, urlMap: Record<string, string>): { value: Record<string, unknown>; count: number } {
166
+ let out: Record<string, unknown> | null = null;
167
+ let count = 0;
168
+ for (const [field, value] of Object.entries(data)) {
169
+ if (typeof value !== "string" && !Array.isArray(value)) continue;
170
+ const r = rewriteSiteUrls(value, urlMap);
171
+ if (!r.count) continue;
172
+ out ??= { ...data };
173
+ out[field] = r.value;
174
+ count += r.count;
175
+ }
176
+ return { value: out ?? data, count };
177
+ }
178
+
179
+ type Content = NonNullable<PluginContext["content"]>;
180
+ type Writable = Content & Required<Pick<Content, "update">>;
181
+
182
+ const versionOf = (item: { version?: number; updatedAt?: string; draftRevisionId?: string | null }) =>
183
+ `${item.version ?? ""}|${item.updatedAt ?? ""}|${item.draftRevisionId ?? ""}`;
184
+
185
+ const PAGE_MS = 12_000;
186
+
187
+ async function scan(ctx: PluginContext, apply: boolean, state: ScanState | null, urlMap: Record<string, string> | null = null) {
188
+ if (!ctx.content || !ctx.schema) throw PluginRouteError.internal("Content access is unavailable.");
189
+ if (apply && !ctx.content.update) throw PluginRouteError.internal("WordPress import needs the content:write capability.");
190
+ const content = ctx.content as Writable;
191
+ const schemas = await ctx.schema.listCollections();
192
+ const info = new Map(schemas.map((c) => [c.slug, c]));
193
+ const collections = state?.collections ?? schemas.filter((c) => c.fields.some((f) => f.type === "portableText")).map((c) => c.slug);
194
+ let index = state?.index ?? 0;
195
+ let cursor = state?.cursor ?? null;
196
+ const opts = await convertOptions(ctx);
197
+ const entries: ScanEntry[] = [];
198
+ const leftovers: Record<string, number> = {};
199
+ let scanned = 0;
200
+ let videosAdded = 0;
201
+ const started = Date.now();
202
+
203
+ while (index < collections.length && Date.now() - started < PAGE_MS) {
204
+ const collection = collections[index] as string;
205
+ const page = await content.list(collection, { limit: 25, ...(cursor ? { cursor } : {}) });
206
+ for (const item of page.items) {
207
+ scanned++;
208
+ const data = await latestData(ctx, collection, item as unknown as { id: string; data: Record<string, unknown>; draftRevisionId?: string | null });
209
+ const convertible = mightNeedConversion(data);
210
+ const rewritable = urlMap !== null && JSON.stringify(data).includes("/wp-content/");
211
+ if (!convertible && !rewritable) continue;
212
+ const result = convertible ? convertEntryData(data, opts) : { value: data, changed: false, changes: [], videos: [], leftovers: {} };
213
+ for (const [k, v] of Object.entries(result.leftovers)) leftovers[k] = (leftovers[k] ?? 0) + v;
214
+ const urls = rewritable ? rewriteEntryUrls(result.value, urlMap) : { value: result.value, count: 0 };
215
+ if (!result.changed && !urls.count) continue;
216
+ result.value = urls.value;
217
+ const changes: Record<string, number> = {};
218
+ for (const c of result.changes) changes[c.to] = (changes[c.to] ?? 0) + 1;
219
+ if (urls.count) changes["media URL"] = urls.count;
220
+ const schema = info.get(collection);
221
+ const titleField = schema?.titleField ?? "title";
222
+ const title = String(data[titleField] ?? data.title ?? item.slug ?? item.id).slice(0, 200);
223
+ const entry: ScanEntry = { collection, id: item.id, title, status: String(item.status ?? ""), changes };
224
+ entries.push(entry);
225
+ if (!apply) continue;
226
+ try {
227
+ const patch: Record<string, unknown> = {};
228
+ for (const [k, v] of Object.entries(result.value)) if (v !== data[k]) patch[k] = v;
229
+ const fresh = await content.get(collection, item.id);
230
+ if (!fresh || versionOf(fresh) !== versionOf(item)) {
231
+ entry.result = "conflict";
232
+ entry.error = "Changed while converting. Run it again.";
233
+ continue;
234
+ }
235
+ const hadDraft = Boolean(item.draftRevisionId);
236
+ await content.update(collection, item.id, patch);
237
+ entry.result = "updated";
238
+ if (schema?.supports.includes("revisions") && item.status === "published") {
239
+ entry.result = "draft";
240
+ if (!hadDraft && content.getVersioned && content.publish) {
241
+ const versioned = await content.getVersioned(collection, item.id);
242
+ // Publish only the draft this update made: exactly one version after the one read.
243
+ if (versioned && versioned.item.version === (fresh.version ?? 0) + 1) {
244
+ await content.publish(collection, item.id, { _rev: versioned._rev });
245
+ entry.result = "published";
246
+ }
247
+ }
248
+ }
249
+ videosAdded += await seedVideos(ctx, result.videos).catch(() => 0);
250
+ } catch (error) {
251
+ entry.result = "failed";
252
+ entry.error = String(error).slice(0, 300);
253
+ ctx.log.error("WordPress import: convert failed", { collection, id: item.id, error: String(error) });
254
+ }
255
+ }
256
+ if (page.hasMore && page.cursor) cursor = page.cursor;
257
+ else {
258
+ index++;
259
+ cursor = null;
260
+ }
261
+ }
262
+ const done = index >= collections.length;
263
+ return {
264
+ done,
265
+ scanned,
266
+ entries,
267
+ leftovers,
268
+ videosAdded,
269
+ state: done ? null : { collections, index, cursor },
270
+ progress: done ? "Done" : `Collection ${Math.min(index + 1, collections.length)} of ${collections.length}`,
271
+ };
272
+ }
273
+
274
+ /** Count the old site's `/wp-content/` URLs in every entry (latest draft included), a page at a time. */
275
+ async function findUrls(ctx: PluginContext, hosts: string[], prefixes: string[], state: ScanState | null) {
276
+ if (!ctx.content || !ctx.schema) throw PluginRouteError.internal("Content access is unavailable.");
277
+ const schemas = await ctx.schema.listCollections();
278
+ const collections = state?.collections ?? schemas.map((c) => c.slug);
279
+ let index = state?.index ?? 0;
280
+ let cursor = state?.cursor ?? null;
281
+ const re = siteUrlPattern(hosts, prefixes);
282
+ const urls = new Map<string, number>();
283
+ let scanned = 0;
284
+ const started = Date.now();
285
+ while (index < collections.length && Date.now() - started < PAGE_MS) {
286
+ const collection = collections[index] as string;
287
+ const page = await ctx.content.list(collection, { limit: 25, ...(cursor ? { cursor } : {}) });
288
+ for (const item of page.items) {
289
+ scanned++;
290
+ const data = await latestData(ctx, collection, item as unknown as { id: string; data: Record<string, unknown>; draftRevisionId?: string | null });
291
+ if (JSON.stringify(data).includes("/wp-content/")) findSiteUrls(data, re, urls);
292
+ }
293
+ if (page.hasMore && page.cursor) cursor = page.cursor;
294
+ else {
295
+ index++;
296
+ cursor = null;
297
+ }
298
+ }
299
+ const done = index >= collections.length;
300
+ return {
301
+ done,
302
+ scanned,
303
+ urls: Object.fromEntries([...urls].slice(0, 20_000)),
304
+ state: done ? null : { collections, index, cursor },
305
+ progress: done ? "Done" : `Collection ${Math.min(index + 1, collections.length)} of ${collections.length}`,
306
+ };
307
+ }
308
+
309
+ // ── Coywolf Files records ────────────────────────────────────────
310
+
311
+ const fileRow = z.object({
312
+ file_id: z.string().regex(WORDPRESS_FILE_ID, "file_id must be the 20-character Coywolf Files id."),
313
+ object_key: z
314
+ .string()
315
+ .min(1)
316
+ .max(512)
317
+ .regex(/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/, "object_key must be a plain bucket key.")
318
+ .refine((k) => !k.split("/").includes(".."), "object_key must be a plain bucket key."),
319
+ filename: z.string().min(1).max(255),
320
+ mime: z.string().max(150).optional().default("application/octet-stream"),
321
+ size: z.coerce.number().int().min(0),
322
+ downloads: z.coerce.number().int().min(0).optional().default(0),
323
+ created: z.string().max(40).optional().default(""),
324
+ });
325
+
326
+ const wpDate = (value: string) => {
327
+ const iso = value && !value.startsWith("0000") ? new Date(`${value.replace(" ", "T")}Z`) : null;
328
+ return iso && !Number.isNaN(iso.getTime()) ? iso.toISOString() : new Date().toISOString();
329
+ };
330
+
331
+ // ── Video Manager library data ───────────────────────────────────
332
+
333
+ const videoLibraryInput = z.object({
334
+ descriptions: z.record(z.string(), z.string().max(5000)).optional(),
335
+ posters: z.record(z.string(), z.object({ mode: z.string().optional(), time: z.coerce.number().optional(), image_url: z.string().max(2000).optional() }).passthrough()).optional(),
336
+ downloads: z.record(z.string(), z.string().max(2000)).optional(),
337
+ });
338
+
339
+ const UID = /^[0-9a-f]{32}$/;
340
+
341
+ export function wpImportPack(): PackModule {
342
+ return {
343
+ id: "wpImport",
344
+ label: "WordPress import",
345
+ features: FEATURES,
346
+ capabilities: ["content:read", "content:write", "schema:read"],
347
+ adminPages: [{ path: "/wordpress-import", label: "WordPress import", icon: "upload-simple" }],
348
+ hooks: { "content:beforeSave": beforeSave },
349
+ routes: {
350
+ "wpImport/settings": {
351
+ permission: "plugins:manage" as const,
352
+ handler: async (ctx: PluginContext) => ({ defaults: normalizeDefaults(await ctx.settings.get(DEFAULTS_SETTING)) }),
353
+ },
354
+ "wpImport/settings/save": definePluginRoute({
355
+ permission: "plugins:manage",
356
+ methods: ["POST"],
357
+ request: { body: "json" },
358
+ handler: async (ctx) => {
359
+ const input = parseInput(z.object({ cvm: z.unknown().optional(), files: z.unknown().optional() }), ctx.input);
360
+ const defaults = defaultsFromWordPress(input.cvm, input.files);
361
+ await ctx.settings.set(DEFAULTS_SETTING, defaults);
362
+ return { defaults };
363
+ },
364
+ }),
365
+ "wpImport/scan": definePluginRoute({
366
+ permission: "plugins:manage",
367
+ methods: ["POST"],
368
+ request: { body: "json", maxBytes: 6 * 1024 * 1024 },
369
+ handler: async (ctx) => {
370
+ await requireFeature(ctx, F.main);
371
+ const input = parseInput(scanInput, ctx.input ?? {});
372
+ const urlMap = input.urlMap && Object.keys(input.urlMap).length ? input.urlMap : null;
373
+ return scan(ctx, input.apply, input.state ?? null, urlMap);
374
+ },
375
+ }),
376
+ "wpImport/urls": definePluginRoute({
377
+ permission: "plugins:manage",
378
+ methods: ["POST"],
379
+ request: { body: "json" },
380
+ handler: async (ctx) => {
381
+ await requireFeature(ctx, F.main);
382
+ const input = parseInput(findInput, ctx.input ?? {});
383
+ return findUrls(ctx, input.hosts, input.prefixes ?? [], input.state ?? null);
384
+ },
385
+ }),
386
+ "wpImport/files": definePluginRoute({
387
+ permission: "plugins:manage",
388
+ methods: ["POST"],
389
+ request: { body: "json", maxBytes: 1024 * 1024 },
390
+ handler: async (ctx) => {
391
+ await requireFeature(ctx, F.main);
392
+ const { text } = parseInput(z.object({ text: z.string().min(1).max(1_000_000) }), ctx.input);
393
+ let rows: Array<Record<string, string>>;
394
+ try {
395
+ rows = parseFileRows(text);
396
+ } catch {
397
+ throw PluginRouteError.badRequest("Paste the tab-separated output of the query, or a JSON array.");
398
+ }
399
+ const uploads = (ctx.storage as Record<string, StorageCollection<UploadDoc> | undefined>)[FILE_COLLECTIONS.uploads];
400
+ const counts = (ctx.storage as Record<string, StorageCollection<{ downloads: number; lastDownload?: string }> | undefined>)[FILE_COLLECTIONS.counts];
401
+ if (!uploads || !counts) throw PluginRouteError.badRequest("Turn on File Downloads first.");
402
+ const added: string[] = [];
403
+ const errors: Array<{ row: number; error: string }> = [];
404
+ for (const [i, raw] of rows.slice(0, 2000).entries()) {
405
+ const parsed = fileRow.safeParse(raw);
406
+ if (!parsed.success) {
407
+ errors.push({ row: i + 1, error: parsed.error.issues[0]?.message ?? "Invalid row" });
408
+ continue;
409
+ }
410
+ const r = parsed.data;
411
+ await uploads.put(r.file_id, { id: r.file_id, key: r.object_key, name: r.filename, size: r.size, type: r.mime || "application/octet-stream", uploadedAt: wpDate(r.created), status: "ready" });
412
+ if (r.downloads > 0 && !(await counts.get(r.file_id))) await counts.put(r.file_id, { downloads: r.downloads });
413
+ invalidateSiteCache(r.file_id);
414
+ added.push(r.file_id);
415
+ }
416
+ return { added: added.length, ids: added, errors };
417
+ },
418
+ }),
419
+ "wpImport/videos": definePluginRoute({
420
+ permission: "plugins:manage",
421
+ methods: ["POST"],
422
+ request: { body: "json", maxBytes: 1024 * 1024 },
423
+ handler: async (ctx) => {
424
+ await requireFeature(ctx, F.main);
425
+ if (!(ctx.storage as Record<string, unknown>).videosMeta) throw PluginRouteError.badRequest("Turn on Videos first.");
426
+ const input = parseInput(videoLibraryInput, ctx.input);
427
+ const store = metaStore(ctx);
428
+ const uids = new Set([...Object.keys(input.descriptions ?? {}), ...Object.keys(input.posters ?? {}), ...Object.keys(input.downloads ?? {})].filter((u) => UID.test(u)));
429
+ let updated = 0;
430
+ for (const uid of uids) {
431
+ const prev = (await store.get(uid)) ?? { uid };
432
+ const next: VideoMeta = { ...prev, uid, updatedAt: new Date().toISOString() };
433
+ const description = input.descriptions?.[uid]?.trim();
434
+ if (description) next.description = description;
435
+ const poster = input.posters?.[uid];
436
+ if (poster?.mode === "image" && poster.image_url && /^https?:\/\//i.test(poster.image_url)) next.posterImage = poster.image_url;
437
+ else if (poster?.mode === "timestamp" && typeof poster.time === "number" && poster.time > 0) next.posterTime = poster.time;
438
+ const download = input.downloads?.[uid];
439
+ if (download && /^https:\/\/\S+\.mp4$/i.test(download)) {
440
+ next.downloadUrl = download;
441
+ next.downloadStatus = "ready";
442
+ }
443
+ await store.put(uid, next);
444
+ updated++;
445
+ }
446
+ return { updated };
447
+ },
448
+ }),
449
+ },
450
+ };
451
+ }
@@ -0,0 +1,202 @@
1
+ /**
2
+ * Category and page parents from a WordPress export (WXR). EmDash's importer
3
+ * creates every category flat (no parent) and drops pages' parents, so
4
+ * WordPress's hierarchical URLs (/news/seo/a-post/, /apps/coywolf-seo/) are
5
+ * lost. The WordPress import page reads them here, then:
6
+ *
7
+ * - categories: sets each term's parent through EmDash's own taxonomy API
8
+ * (plugins can read terms but not update them), after a dry run
9
+ * (planCategoryParents);
10
+ * - pages: EmDash entries have no parent, so it prints a `pageParents` map
11
+ * for coywolfPlugin() (used by the `{pagepath}` URL token).
12
+ *
13
+ * In the export, `<wp:category>` holds `<wp:category_nicename>` (slug) and
14
+ * `<wp:category_parent>` (the parent's slug); `<wp:term>` entries with
15
+ * `<wp:term_taxonomy>category</wp:term_taxonomy>` use `<wp:term_slug>` and
16
+ * `<wp:term_parent>`. Pages are `<item>`s with `<wp:post_type>page</wp:post_type>`
17
+ * and `<wp:post_parent>` (the parent's post id; 0 = none).
18
+ *
19
+ * Pure, no imports: runs in the admin (browser), a Node script and tests.
20
+ */
21
+
22
+ const CDATA = /<!\[CDATA\[([\s\S]*?)\]\]>/g;
23
+
24
+ function text(body: string): string {
25
+ if (!body.includes("<![CDATA[")) return body.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&amp;/g, "&");
26
+ let out = "";
27
+ for (const m of body.matchAll(CDATA)) out += m[1];
28
+ return out;
29
+ }
30
+
31
+ function tag(block: string, name: string): string {
32
+ const m = block.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
33
+ return m ? text(m[1] as string).trim() : "";
34
+ }
35
+
36
+ /** WordPress stores slugs percent-encoded (non-ASCII); EmDash keeps the decoded form. */
37
+ function slugText(raw: string): string {
38
+ try {
39
+ return decodeURIComponent(raw);
40
+ } catch {
41
+ return raw;
42
+ }
43
+ }
44
+
45
+ /** WordPress stores term names HTML-escaped (`Tips &amp; Tricks`). */
46
+ const decodeName = (name: string) => name.replace(/&quot;/g, '"').replace(/&#0?39;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&");
47
+
48
+ export interface WxrCategory {
49
+ slug: string;
50
+ name: string;
51
+ /** The parent's slug, "" for a top-level category. */
52
+ parent: string;
53
+ }
54
+
55
+ /** Every category in an export, with its parent's slug. */
56
+ export function wxrCategories(xml: string): WxrCategory[] {
57
+ const out = new Map<string, WxrCategory>();
58
+ for (const m of xml.matchAll(/<wp:category>([\s\S]*?)<\/wp:category>/g)) {
59
+ const block = m[1] as string;
60
+ const slug = slugText(tag(block, "wp:category_nicename"));
61
+ if (slug && !out.has(slug)) out.set(slug, { slug, name: decodeName(tag(block, "wp:cat_name")) || slug, parent: slugText(tag(block, "wp:category_parent")) });
62
+ }
63
+ for (const m of xml.matchAll(/<wp:term>([\s\S]*?)<\/wp:term>/g)) {
64
+ const block = m[1] as string;
65
+ if (tag(block, "wp:term_taxonomy") !== "category") continue;
66
+ const slug = slugText(tag(block, "wp:term_slug"));
67
+ if (slug && !out.has(slug)) out.set(slug, { slug, name: decodeName(tag(block, "wp:term_name")) || slug, parent: slugText(tag(block, "wp:term_parent")) });
68
+ }
69
+ return [...out.values()];
70
+ }
71
+
72
+ export interface WxrPage {
73
+ id: number;
74
+ slug: string;
75
+ title: string;
76
+ /** The parent page's slug, "" for a top-level page (or a parent not in the export). */
77
+ parent: string;
78
+ }
79
+
80
+ /** Pages in an export (not trashed or auto-drafts), with their parent page's slug. */
81
+ export function wxrPages(xml: string): WxrPage[] {
82
+ const raw: Array<{ id: number; slug: string; title: string; parentId: number }> = [];
83
+ for (const m of xml.matchAll(/<item>([\s\S]*?)<\/item>/g)) {
84
+ const item = m[1] as string;
85
+ if (tag(item, "wp:post_type") !== "page") continue;
86
+ const status = tag(item, "wp:status");
87
+ if (status === "trash" || status === "auto-draft" || status === "inherit") continue;
88
+ const slug = slugText(tag(item, "wp:post_name"));
89
+ if (!slug) continue;
90
+ raw.push({ id: Number(tag(item, "wp:post_id")) || 0, slug, title: tag(item, "title"), parentId: Number(tag(item, "wp:post_parent")) || 0 });
91
+ }
92
+ const slugById = new Map(raw.filter((p) => p.id).map((p) => [p.id, p.slug]));
93
+ return raw.map((p) => ({ id: p.id, slug: p.slug, title: p.title, parent: (p.parentId && slugById.get(p.parentId)) || "" }));
94
+ }
95
+
96
+ /** slug → parent slug, for the entries that have a parent (and aren't their own parent). */
97
+ export function parentsMap(items: Array<{ slug: string; parent: string }>): Record<string, string> {
98
+ const out: Record<string, string> = {};
99
+ for (const item of items) if (item.parent && item.parent !== item.slug && !Object.hasOwn(out, item.slug)) out[item.slug] = item.parent;
100
+ return out;
101
+ }
102
+
103
+ /** A term as EmDash's taxonomy API lists it (flattened). */
104
+ export interface SiteTerm {
105
+ id: string;
106
+ slug: string;
107
+ label?: string;
108
+ /** The parent's translation_group (or row id), null at the top level. */
109
+ parentId: string | null;
110
+ translationGroup?: string | null;
111
+ locale?: string;
112
+ children?: SiteTerm[];
113
+ }
114
+
115
+ /** EmDash lists hierarchical taxonomies as a tree: flatten it (first row per slug wins). */
116
+ export function flattenTerms(terms: SiteTerm[]): SiteTerm[] {
117
+ const out = new Map<string, SiteTerm>();
118
+ const walk = (nodes: SiteTerm[], depth: number) => {
119
+ if (depth > 64) return;
120
+ for (const node of nodes) {
121
+ if (!out.has(node.slug)) out.set(node.slug, node);
122
+ if (node.children?.length) walk(node.children, depth + 1);
123
+ }
124
+ };
125
+ walk(terms, 0);
126
+ return [...out.values()];
127
+ }
128
+
129
+ export interface ParentPlan {
130
+ slug: string;
131
+ name: string;
132
+ /** The parent WordPress had. */
133
+ parent: string;
134
+ /** The parent on the site now ("" = none). */
135
+ current: string;
136
+ /** The site's term id (to update) and the parent term's id (to set). */
137
+ termId?: string;
138
+ parentTermId?: string;
139
+ /** set: the parent will be set; done: already right; missing: the term or its parent isn't on the site. */
140
+ action: "set" | "done" | "missing";
141
+ note?: string;
142
+ result?: string;
143
+ }
144
+
145
+ /**
146
+ * What restoring WordPress's category parents would change, per category
147
+ * with a parent in WordPress. Categories that are already right are "done",
148
+ * so running it again changes nothing.
149
+ */
150
+ export function planCategoryParents(categories: WxrCategory[], siteTerms: SiteTerm[]): ParentPlan[] {
151
+ const terms = flattenTerms(siteTerms);
152
+ const bySlug = new Map(terms.map((t) => [t.slug, t]));
153
+ const slugByRef = new Map<string, string>();
154
+ for (const t of terms) {
155
+ slugByRef.set(t.id, t.slug);
156
+ if (t.translationGroup && !slugByRef.has(t.translationGroup)) slugByRef.set(t.translationGroup, t.slug);
157
+ }
158
+ const plans: ParentPlan[] = [];
159
+ for (const c of categories) {
160
+ if (!c.parent || c.parent === c.slug) continue;
161
+ const term = bySlug.get(c.slug);
162
+ const parent = bySlug.get(c.parent);
163
+ const current = term?.parentId ? (slugByRef.get(term.parentId) ?? term.parentId) : "";
164
+ const plan: ParentPlan = { slug: c.slug, name: c.name, parent: c.parent, current, termId: term?.id, parentTermId: parent?.id, action: "set" };
165
+ if (!term) {
166
+ plan.action = "missing";
167
+ plan.note = `No category “${c.slug}” on the site`;
168
+ } else if (!parent) {
169
+ plan.action = "missing";
170
+ plan.note = `The parent “${c.parent}” isn't on the site`;
171
+ } else if (current === c.parent) {
172
+ plan.action = "done";
173
+ }
174
+ plans.push(plan);
175
+ }
176
+ // Parents before children, so each update finds its parent in place.
177
+ const depth = (slug: string) => {
178
+ let d = 0;
179
+ const seen = new Set<string>();
180
+ let up = categories.find((c) => c.slug === slug)?.parent;
181
+ while (up && !seen.has(up) && d < 64) {
182
+ seen.add(up);
183
+ d++;
184
+ up = categories.find((c) => c.slug === up)?.parent;
185
+ }
186
+ return d;
187
+ };
188
+ return plans.sort((a, b) => depth(a.slug) - depth(b.slug) || a.slug.localeCompare(b.slug));
189
+ }
190
+
191
+ /** A ready-to-paste coywolfPlugin() option, e.g. `pageParents: { "coywolf-seo": "apps" },`. */
192
+ export function optionSnippet(name: string, map: Record<string, string>): string {
193
+ const entries = Object.entries(map).sort((a, b) => a[1].localeCompare(b[1]) || a[0].localeCompare(b[0]));
194
+ if (!entries.length) return `${name}: {},`;
195
+ return `${name}: {\n${entries.map(([k, v]) => `\t${JSON.stringify(k)}: ${JSON.stringify(v)},`).join("\n")}\n},`;
196
+ }
197
+
198
+ /** The same as a `termParents` option for one taxonomy: `termParents: { category: { … } },`. */
199
+ export function termParentsSnippet(taxonomy: string, map: Record<string, string>): string {
200
+ const inner = optionSnippet(taxonomy, map).replace(/,$/, "").replace(/\n/g, "\n\t");
201
+ return `termParents: {\n\t${inner},\n},`;
202
+ }