@kidecms/wordpress 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/plan.ts ADDED
@@ -0,0 +1,1467 @@
1
+ /**
2
+ * `kide import plan`: run every mapping over the staged source and produce
3
+ * - the documents to write (validated, hashed, media as placeholder tokens),
4
+ * - a coverage ledger: every source path mapped, dropped with a reason, or unaccounted,
5
+ * - integrity checks: dangling refs, missing media, text recall per record.
6
+ * Nothing touches the CMS database; `apply` writes what the plan stored.
7
+ */
8
+ import { createHash } from "node:crypto";
9
+ import { existsSync, readdirSync } from "node:fs";
10
+ import path from "node:path";
11
+ import { pathToFileURL } from "node:url";
12
+
13
+ import {
14
+ decodeEntities,
15
+ documentRoute,
16
+ getTranslatableFieldNames,
17
+ htmlToRichText,
18
+ slugify,
19
+ type CMSConfig,
20
+ type CollectionConfig,
21
+ type FieldConfig,
22
+ type ContentNode,
23
+ type RichTextNode,
24
+ validateDocument,
25
+ validateTranslations,
26
+ } from "@kidecms/core";
27
+ import {
28
+ CONTAINER_BLOCKS,
29
+ CONVERTED,
30
+ EMPTY_BLOCKS,
31
+ HTML_BLOCKS,
32
+ blockData,
33
+ collectDomain,
34
+ expandBlocks,
35
+ htmlText,
36
+ isCovered,
37
+ isEmpty,
38
+ matchesPath,
39
+ outputStrings,
40
+ stripComments,
41
+ textRecall,
42
+ truncate,
43
+ words,
44
+ } from "./domain";
45
+ import {
46
+ autop,
47
+ blockHtml,
48
+ findShortcodes,
49
+ isAcfBlock,
50
+ replaceShortcodes,
51
+ walkBlocks,
52
+ type ShortcodeHandler,
53
+ type WpBlock,
54
+ } from "./gutenberg";
55
+ import {
56
+ isSkip,
57
+ Tracker,
58
+ track,
59
+ type BlockInput,
60
+ type MapContext,
61
+ type Mapping,
62
+ type MappingModule,
63
+ type SourceRecord,
64
+ type Skip,
65
+ } from "./mapping";
66
+ import { reusableLookup } from "./profile";
67
+ import { autoFields, autoRecord } from "./automap";
68
+ import { gravityForm, menuItems, termTree } from "./structures";
69
+ import { termParts } from "./terms";
70
+ import { getMeta, loadMedia, loadRecords, type Staging, type StagedMedia, type StagedRecord } from "./staging";
71
+
72
+ /** Synthetic source type: one record per URL the plan changes, for redirect mappings. */
73
+ export const URL_CHANGES = "url-change";
74
+
75
+ export const MEDIA_PREFIX = "/uploads/.kide-media/";
76
+ export const MEDIA_TOKEN = /\/uploads\/\.kide-media\/([A-Za-z0-9_-]+)/g;
77
+ export const mediaToken = (key: string) => `${MEDIA_PREFIX}${Buffer.from(key).toString("base64url")}`;
78
+ export const tokenKey = (encoded: string) => Buffer.from(encoded, "base64url").toString("utf8");
79
+
80
+ /** `acf/color-text` / `exf/link-list` → `colorText` / `linkList`: the Kide block a block maps onto by name. */
81
+ const blockTypeOf = (name: string) =>
82
+ name.slice(name.indexOf("/") + 1).replace(/[-_]+([a-z0-9])/g, (_, c: string) => c.toUpperCase());
83
+
84
+ /** Markup that means something without text: embeds, media, forms, custom elements. */
85
+ const MARKUP = /<(iframe|script|video|audio|object|embed|img|svg|form|input|canvas|[a-z]+-[a-z][\w-]*)\b/i;
86
+ const DEFAULT_SHORTCODES: Record<string, ShortcodeHandler> = {
87
+ caption: (_, inner) => inner,
88
+ embed: (_, inner) => inner,
89
+ };
90
+
91
+ export type IssueKind =
92
+ | "invalid"
93
+ | "shape"
94
+ | "error"
95
+ | "dangling-ref"
96
+ | "missing-media"
97
+ | "low-recall"
98
+ | "duplicate-id"
99
+ | "lost-translation"
100
+ | "translation-advice"
101
+ | "url-change"
102
+ | "dangling-term"
103
+ | "bundled-term"
104
+ | "warning";
105
+ const BLOCKING: IssueKind[] = [
106
+ "invalid",
107
+ "shape",
108
+ "error",
109
+ "dangling-ref",
110
+ "dangling-term",
111
+ "bundled-term",
112
+ "missing-media",
113
+ "low-recall",
114
+ "duplicate-id",
115
+ ];
116
+
117
+ /** Compare URLs the way a visitor experiences them: decoded, NFC, no query and no trailing slash. */
118
+ export const normalizePath = (path: string): string => {
119
+ let p = path.split(/[?#]/)[0];
120
+ try {
121
+ p = decodeURI(p);
122
+ } catch {}
123
+ p = p.normalize("NFC");
124
+ if (p.length > 1) p = p.replace(/\/+$/, "");
125
+ return p || "/";
126
+ };
127
+ /** Text editors translate; anything else (colour, image, date, relation) is usually the same in every language. */
128
+ const isTranslatedText = (field: { type: string; admin?: { component?: string } }) =>
129
+ ["text", "slug", "richText", "content", "blocks"].includes(field.type)
130
+ ? !(field.type === "text" && field.admin?.component)
131
+ : ["taxonomy-terms", "menu-items"].includes(field.admin?.component ?? "");
132
+
133
+ export type Issue = { kind: IssueKind; key?: string; message: string };
134
+
135
+ export type PathStatus = {
136
+ path: string;
137
+ records: number;
138
+ sample: string;
139
+ status: "mapped" | "dropped" | "unaccounted";
140
+ reason?: string;
141
+ };
142
+
143
+ export type SourceLedger = {
144
+ source: string;
145
+ collection?: string;
146
+ skip?: string;
147
+ records: number;
148
+ filtered: Record<string, number>;
149
+ docs: number;
150
+ paths: PathStatus[];
151
+ issues: Issue[];
152
+ recall: { min: number; avg: number };
153
+ staleDrops: string[];
154
+ };
155
+
156
+ export type PlannedDoc = {
157
+ docId: string;
158
+ collection: string;
159
+ sourceKeys: string[];
160
+ data: Record<string, unknown>;
161
+ translations: Record<string, Record<string, unknown>>;
162
+ /** Expected text per locale ("base" = the base row), for verify. */
163
+ expected: Record<string, string[]>;
164
+ media: string[];
165
+ /** How verify compares text: the mapping's `textMatch`. */
166
+ textMatch: "multiset" | "set";
167
+ hash: string;
168
+ };
169
+
170
+ export type UrlChange = { key: string; collection: string; docId: string; from: string; to: string };
171
+
172
+ export type PlanResult = {
173
+ urlChanges: UrlChange[];
174
+ ledgers: SourceLedger[];
175
+ unmapped: Array<{ type: string; records: number }>;
176
+ docs: PlannedDoc[];
177
+ media: Map<string, StagedMedia>;
178
+ blocking: string[];
179
+ durationMs: number;
180
+ };
181
+
182
+ export const mappingsDir = () =>
183
+ path.resolve(process.env.KIDE_IMPORT_MAPPINGS ?? path.join(process.cwd(), "migration"));
184
+
185
+ export const loadMappings = async (dir = mappingsDir()): Promise<Array<Mapping | Skip>> => {
186
+ if (!existsSync(dir)) return [];
187
+ const files = readdirSync(dir)
188
+ .filter((f) => /\.map\.(ts|mts|js|mjs)$/.test(f))
189
+ .sort();
190
+ const out: Array<Mapping | Skip> = [];
191
+ for (const file of files) {
192
+ const mod = (await import(pathToFileURL(path.join(dir, file)).href)).default as MappingModule;
193
+ out.push(...(Array.isArray(mod) ? mod : [mod]));
194
+ }
195
+ const skipped = new Set(out.filter(isSkip).map((m) => m.source));
196
+ const conflicts = [...skipped].filter((s) => out.filter((m) => m.source === s).length > 1);
197
+ if (conflicts.length) throw new Error(`Source type(s) both skipped and mapped: ${conflicts.join(", ")}`);
198
+ return out;
199
+ };
200
+
201
+ /** `wp-12` for posts, `wp-term-5` for terms, `wp-redirect-3` / `wp-form-4` / … for every other source. */
202
+ const defaultId = (r: StagedRecord) =>
203
+ r.key.startsWith("post:") ? `wp-${r.id}` : `wp-${r.key.replace(/[^A-Za-z0-9]+/g, "-").toLowerCase()}`;
204
+
205
+ const sha = (value: unknown) => createHash("sha1").update(JSON.stringify(value)).digest("hex");
206
+
207
+ const firstDrop = (mapping: Mapping, p: string): [string, string] | undefined => {
208
+ for (const [pattern, reason] of Object.entries(mapping.drop ?? {}))
209
+ if (matchesPath(pattern, p)) return [pattern, reason];
210
+ const contentDrop = (p.startsWith("block:") || p.startsWith("shortcode:")) && mapping.drop?.content;
211
+ return contentDrop ? ["content", contentDrop] : undefined;
212
+ };
213
+
214
+ export const planImport = async (
215
+ staging: Staging,
216
+ config: CMSConfig,
217
+ mappings: Array<Mapping | Skip>,
218
+ options: { minRecall?: number; only?: string[] } = {},
219
+ ): Promise<PlanResult> => {
220
+ const started = Date.now();
221
+ const minRecall = options.minRecall ?? 0.95;
222
+ const records = loadRecords(staging);
223
+ const mediaList = loadMedia(staging);
224
+ const siteUrl = getMeta(staging, "siteUrl") ?? "";
225
+ const uploadsUrl = getMeta(staging, "uploadsUrl") ?? `${siteUrl}/wp-content/uploads`;
226
+ const uploadsDir = getMeta(staging, "uploadsDir");
227
+ const uploadsPath = (() => {
228
+ try {
229
+ return new URL(uploadsUrl).pathname.replace(/\/$/, "");
230
+ } catch {
231
+ return "/wp-content/uploads";
232
+ }
233
+ })();
234
+ const siteHosts = new Set(
235
+ [siteUrl, uploadsUrl].flatMap((u) => {
236
+ try {
237
+ const host = new URL(u).host;
238
+ return [host, host.startsWith("www.") ? host.slice(4) : `www.${host}`];
239
+ } catch {
240
+ return [];
241
+ }
242
+ }),
243
+ );
244
+
245
+ const reusable = reusableLookup(records);
246
+ const byKey = new Map(records.map((r) => [r.key, r]));
247
+ const byType = new Map<string, StagedRecord[]>();
248
+ for (const r of records) {
249
+ const list = byType.get(r.type);
250
+ if (list) list.push(r);
251
+ else byType.set(r.type, [r]);
252
+ }
253
+ const media = new Map(mediaList.map((m) => [m.key, m]));
254
+ const byGroup = new Map<string, StagedRecord[]>();
255
+ for (const r of records) {
256
+ if (!r.group) continue;
257
+ const members = byGroup.get(r.group);
258
+ if (members) members.push(r);
259
+ else byGroup.set(r.group, [r]);
260
+ }
261
+ const fileIndex = new Map(mediaList.filter((m) => m.file).map((m) => [m.file!, m.key]));
262
+ const collections = new Map(config.collections.map((c) => [c.slug, c]));
263
+ const defaultLocale = config.locales?.default;
264
+
265
+ const active = mappings.filter((m) => !options.only || options.only.includes(m.source));
266
+ const docIdByKey = new Map<string, string>();
267
+ const collectionByKey = new Map<string, string>();
268
+ const sourceCounts = new Map<string, number>();
269
+ for (const m of mappings) if (!isSkip(m)) sourceCounts.set(m.source, (sourceCounts.get(m.source) ?? 0) + 1);
270
+ /** Sources split over several mappings: a filter returning `false` there means "another mapping's record". */
271
+ const shared = new Set([...sourceCounts].filter(([, n]) => n > 1).map(([s]) => s));
272
+ const ledgers: SourceLedger[] = [];
273
+ const docs: PlannedDoc[] = [];
274
+ const urlChanges: UrlChange[] = [];
275
+ /** Every route a planned document is served at, so redirects can't hide one. */
276
+ const routes = new Set<string>();
277
+ const docLedger = new Map<string, SourceLedger>();
278
+
279
+ // --- pass 1: filter, group translations, assign document ids -----------------
280
+ type Prepared = {
281
+ mapping: Mapping;
282
+ collection: CollectionConfig;
283
+ ledger: SourceLedger;
284
+ groups: StagedRecord[][];
285
+ filterReads: Set<string>;
286
+ };
287
+ const prepared: Prepared[] = [];
288
+ const allMappedIds = new Map<string, string>();
289
+ const accounted = new Set<string>();
290
+ const splitProblems: string[] = [];
291
+ for (const m of mappings) {
292
+ if (isSkip(m)) continue;
293
+ for (const r of byType.get(m.source) ?? []) {
294
+ const verdict = m.filter ? m.filter(r) : true;
295
+ if (verdict !== false) accounted.add(r.key);
296
+ if (verdict !== true) continue;
297
+ if (collectionByKey.has(r.key))
298
+ splitProblems.push(`\`${r.key}\` is claimed by both \`${collectionByKey.get(r.key)}\` and \`${m.collection}\``);
299
+ allMappedIds.set(r.key, "");
300
+ collectionByKey.set(r.key, m.collection);
301
+ }
302
+ }
303
+ for (const source of shared) {
304
+ const lost = (byType.get(source) ?? []).filter((r) => !accounted.has(r.key));
305
+ if (lost.length)
306
+ splitProblems.push(
307
+ `\`${source}\`: ${lost.length} record(s) no mapping's filter claims (e.g. ${lost
308
+ .slice(0, 3)
309
+ .map((r) => r.key)
310
+ .join(", ")}) — return a reason string to leave one out`,
311
+ );
312
+ }
313
+
314
+ const prepareMapping = (m: Mapping | Skip) => {
315
+ const list = byType.get(m.source) ?? [];
316
+ const ledger: SourceLedger = {
317
+ source: m.source,
318
+ records: list.length,
319
+ filtered: {},
320
+ docs: 0,
321
+ paths: [],
322
+ issues: [],
323
+ recall: { min: 1, avg: 1 },
324
+ staleDrops: [],
325
+ };
326
+ ledgers.push(ledger);
327
+ if (isSkip(m)) {
328
+ ledger.skip = m.skip;
329
+ return;
330
+ }
331
+ ledger.collection = m.collection;
332
+ const collection = collections.get(m.collection);
333
+ if (!collection) {
334
+ ledger.issues.push({
335
+ kind: "invalid",
336
+ message: `unknown collection "${m.collection}" — declare it in cms.config.ts`,
337
+ });
338
+ return;
339
+ }
340
+ if (!list.length) ledger.issues.push({ kind: "warning", message: `no staged records of type "${m.source}"` });
341
+ const filterReads = new Set<string>();
342
+ const filterTracker = new Tracker(filterReads);
343
+ const kept: StagedRecord[] = [];
344
+ for (const r of list) {
345
+ const verdict = m.filter ? m.filter(track(r, "", filterTracker) as SourceRecord) : true;
346
+ if (verdict === true) kept.push(r);
347
+ else if (verdict === false && shared.has(m.source)) ledger.records--;
348
+ else {
349
+ const reason = typeof verdict === "string" ? verdict : "filtered";
350
+ ledger.filtered[reason] = (ledger.filtered[reason] ?? 0) + 1;
351
+ }
352
+ }
353
+ // Editors sometimes never link a page to its translation; one unlinked record per locale
354
+ // sharing a slug is that pair.
355
+ const unlinked = new Map<string, StagedRecord[]>();
356
+ for (const r of kept)
357
+ if (!r.group && r.locale && typeof r.slug === "string" && r.slug)
358
+ unlinked.set(r.slug, [...(unlinked.get(r.slug) ?? []), r]);
359
+ const pairedGroup = new Map<string, string>();
360
+ for (const [slug, records] of unlinked) {
361
+ const locales = new Set(records.map((r) => r.locale));
362
+ if (records.length < 2 || locales.size !== records.length) continue;
363
+ for (const r of records) pairedGroup.set(r.key, `paired:${slug}`);
364
+ ledger.issues.push({
365
+ kind: "warning",
366
+ key: records[0].key,
367
+ message: `${records.map((r) => r.key).join(" + ")} aren't linked as translations in the source but share the slug "${slug}"; imported as one document`,
368
+ });
369
+ }
370
+ const grouped = new Map<string, StagedRecord[]>();
371
+ for (const r of kept) {
372
+ const g = r.group ?? pairedGroup.get(r.key) ?? r.key;
373
+ const members = grouped.get(g);
374
+ if (members) members.push(r);
375
+ else grouped.set(g, [r]);
376
+ }
377
+ const groups = [...grouped.values()].map((members) => {
378
+ members.sort((a, b) => a.id - b.id);
379
+ // The original is the version written first (WordPress ids grow with time), not the
380
+ // site's default language; synthetic records without real ids fall back to the default.
381
+ const dated = members.every((x) => x.id > 0) && new Set(members.map((x) => x.id)).size === members.length;
382
+ const base = dated ? members[0] : (members.find((x) => x.locale === defaultLocale) ?? members[0]);
383
+ return [base, ...members.filter((x) => x !== base)];
384
+ });
385
+ for (const [base, ...rest] of groups) {
386
+ const docId = m.id ? m.id(base) : defaultId(base);
387
+ for (const r of [base, ...rest]) docIdByKey.set(r.key, docId);
388
+ }
389
+ prepared.push({ mapping: m, collection, ledger, groups, filterReads });
390
+ };
391
+ for (const m of active) if (m.source !== URL_CHANGES) prepareMapping(m);
392
+
393
+ // Refs into mappings excluded by `--only` still resolve, so partial plans don't report false dangling refs.
394
+ for (const m of mappings) {
395
+ if (isSkip(m) || active.includes(m)) continue;
396
+ for (const r of byType.get(m.source) ?? []) {
397
+ if (!allMappedIds.has(r.key)) continue;
398
+ docIdByKey.set(r.key, m.id ? m.id(r) : defaultId(r));
399
+ }
400
+ }
401
+
402
+ // --- media resolution ------------------------------------------------------
403
+ const ensureMedia = (key: string, init: () => StagedMedia) => {
404
+ if (!media.has(key)) media.set(key, init());
405
+ return key;
406
+ };
407
+ const urlToMediaKey = (raw: string, allowExternal: boolean): string | null => {
408
+ let u = raw.trim().split(/[?#]/)[0];
409
+ if (u.startsWith("//")) u = `https:${u}`;
410
+ const idx = u.indexOf(`${uploadsPath}/`);
411
+ if (idx >= 0 && (u.startsWith("/") || siteHosts.has(hostOf(u)) || /\/wp-content\/uploads\//.test(u))) {
412
+ const rel = decodeURIComponent(u.slice(idx + uploadsPath.length + 1));
413
+ const stripped = rel.replace(/-\d+x\d+(\.\w+)$/, "$1");
414
+ for (const candidate of [rel, stripped, stripped.replace(/-scaled(\.\w+)$/, "$1")]) {
415
+ const key = fileIndex.get(candidate);
416
+ if (key) return key;
417
+ }
418
+ return ensureMedia(`file:${rel}`, () => ({
419
+ key: `file:${rel}`,
420
+ attachmentId: null,
421
+ file: rel,
422
+ url: `${uploadsUrl}/${rel}`,
423
+ mime: null,
424
+ alt: null,
425
+ }));
426
+ }
427
+ if (allowExternal && /^https?:\/\//.test(u))
428
+ return ensureMedia(`url:${u}`, () => ({
429
+ key: `url:${u}`,
430
+ attachmentId: null,
431
+ file: null,
432
+ url: u,
433
+ mime: null,
434
+ alt: null,
435
+ }));
436
+ return null;
437
+ };
438
+ const urlIndex = new Map<string, string>();
439
+ for (const r of records) if (typeof r.permalink === "string") urlIndex.set(normalizePath(r.permalink), r.key);
440
+ const sitePath = (url: string): string | null => {
441
+ if (url.startsWith("/") && !url.startsWith("//")) return url;
442
+ try {
443
+ const parsed = new URL(url.startsWith("//") ? `https:${url}` : url);
444
+ return siteHosts.has(parsed.host) ? `${parsed.pathname}${parsed.search}${parsed.hash}` || "/" : null;
445
+ } catch {
446
+ return null;
447
+ }
448
+ };
449
+ /** The staged record an internal URL points at (by its WordPress permalink). */
450
+ const recordForUrl = (url: string): StagedRecord | undefined => {
451
+ const p = sitePath(url);
452
+ const key = p ? urlIndex.get(normalizePath(p)) : undefined;
453
+ return key ? byKey.get(key) : undefined;
454
+ };
455
+ const internalUrl = (url: string): string => {
456
+ const p = sitePath(url);
457
+ if (p === null) return url;
458
+ const target = recordForUrl(url);
459
+ return target && docIdByKey.has(target.key) ? normalizePath(p) : p;
460
+ };
461
+
462
+ // --- pass 2: map --------------------------------------------------------------
463
+ const mapPrepared = ({ mapping, collection, ledger, groups, filterReads }: Prepared) => {
464
+ const reads = new Set(filterReads);
465
+ const domain = new Map<string, { records: number; sample: string }>();
466
+ const recalls: number[] = [];
467
+ const translatable = new Set(getTranslatableFieldNames(collection));
468
+ const declared = Object.assign(
469
+ {},
470
+ ...Object.values(collection.fields).map((f) => (f.type === "content" ? (f.blocks ?? {}) : {})),
471
+ );
472
+ const notices = new Set<string>();
473
+ // Translatable fields a translation leaves empty: Kide serves the original's value there.
474
+ const inherits = new Map<string, string[]>();
475
+ // Per field: how many translations exist, and how many carry a value of their own.
476
+ const divergence = new Map<string, { translations: number; differs: number; samples: string[] }>();
477
+ for (const type of mapping.includes ?? []) {
478
+ for (const r of byType.get(type) ?? []) {
479
+ const seen = new Set<string>();
480
+ collectDomain(r, reusable, (p, value) => {
481
+ const key = `${type}:${p}`;
482
+ const stat = domain.get(key) ?? { records: 0, sample: truncate(value) };
483
+ if (!seen.has(key)) {
484
+ seen.add(key);
485
+ stat.records++;
486
+ }
487
+ domain.set(key, stat);
488
+ });
489
+ }
490
+ }
491
+
492
+ for (const members of groups) {
493
+ for (const r of members) {
494
+ const seen = new Set<string>();
495
+ collectDomain(r, reusable, (p, value) => {
496
+ const stat = domain.get(p) ?? { records: 0, sample: truncate(value) };
497
+ if (!seen.has(p)) {
498
+ seen.add(p);
499
+ stat.records++;
500
+ }
501
+ domain.set(p, stat);
502
+ });
503
+ }
504
+
505
+ const docId = docIdByKey.get(members[0].key)!;
506
+ const docMedia = new Set<string>();
507
+ const outputs: Array<{ record: StagedRecord; out: Record<string, unknown>; tracker: Tracker }> = [];
508
+ for (const record of members) {
509
+ const tracker = new Tracker(reads);
510
+ const ctx = makeContext({
511
+ mapping,
512
+ record,
513
+ tracker,
514
+ issues: ledger.issues,
515
+ docMedia,
516
+ docIdByKey,
517
+ collectionByKey,
518
+ byKey,
519
+ media,
520
+ fileIndex,
521
+ reusable,
522
+ uploadsDir,
523
+ urlToMediaKey,
524
+ internalUrl,
525
+ recordForUrl,
526
+ byType,
527
+ byGroup,
528
+ collection,
529
+ declared,
530
+ notices,
531
+ });
532
+ try {
533
+ const out = mapping.map(track(record, "", tracker) as any, ctx);
534
+ outputs.push({ record, out: JSON.parse(JSON.stringify(out ?? {})), tracker });
535
+ } catch (e) {
536
+ ledger.issues.push({ kind: "error", key: record.key, message: `map() threw: ${(e as Error).message}` });
537
+ }
538
+ }
539
+ if (!outputs.length || outputs[0].record !== members[0]) continue;
540
+
541
+ const [base, ...rest] = outputs;
542
+ const data: Record<string, unknown> = { ...base.out, _id: docId };
543
+ if (collection.drafts && data._status === undefined)
544
+ data._status = ["publish", "private"].includes(base.record.status) ? "published" : "draft";
545
+ if (config.locales && translatable.size && base.record.locale && data._sourceLocale === undefined)
546
+ data._sourceLocale = base.record.locale;
547
+
548
+ const translations: Record<string, Record<string, unknown>> = {};
549
+ const expected: Record<string, string[]> = { base: base.tracker.expectedTexts() };
550
+ for (const { record, out, tracker } of rest) {
551
+ if (!record.locale || record.locale === base.record.locale) {
552
+ ledger.issues.push({
553
+ kind: "warning",
554
+ key: record.key,
555
+ message: `translation group member without a distinct locale (${record.locale ?? "none"}) — not imported`,
556
+ });
557
+ continue;
558
+ }
559
+ // A value equal to the original is inherited, not copied: a copy would go stale when the original changes.
560
+ const overlay: Record<string, unknown> = {};
561
+ for (const k of new Set([...Object.keys(out), ...Object.keys(base.out)])) {
562
+ if (k.startsWith("_")) continue;
563
+ const differs = JSON.stringify(out[k] ?? null) !== JSON.stringify(base.out[k] ?? null);
564
+ // Two empty values say nothing about whether the field is shared.
565
+ if (!differs && isEmpty(out[k])) continue;
566
+ if (isEmpty(out[k]) && translatable.has(k) && isTranslatedText(collection.fields[k]))
567
+ inherits.set(k, [...(inherits.get(k) ?? []), record.key]);
568
+ const stat = divergence.get(k) ?? { translations: 0, differs: 0, samples: [] };
569
+ stat.translations++;
570
+ if (differs) {
571
+ stat.differs++;
572
+ if (stat.samples.length < 3) stat.samples.push(record.key);
573
+ }
574
+ divergence.set(k, stat);
575
+ // Slugs stay explicit: an empty one is regenerated from the translated title.
576
+ const keep = differs || collection.fields[k]?.type === "slug";
577
+ if (translatable.has(k) && keep && k in out) overlay[k] = out[k];
578
+ }
579
+ translations[record.locale] = overlay;
580
+ expected[record.locale] = tracker.expectedTexts();
581
+ }
582
+
583
+ const doc = validateDocument(collection, data, config);
584
+ const i18n = validateTranslations(
585
+ config,
586
+ collection,
587
+ Object.keys(translations).length ? translations : undefined,
588
+ data._sourceLocale,
589
+ );
590
+ for (const e of [...doc.errors, ...i18n.errors])
591
+ ledger.issues.push({ kind: "invalid", key: base.record.key, message: `${e.field}: ${e.message}` });
592
+ for (const w of [...doc.warnings, ...i18n.warnings])
593
+ ledger.issues.push({ kind: "shape", key: base.record.key, message: `${w.field}: ${w.message}` });
594
+
595
+ for (const { record } of outputs) {
596
+ const locale = record === base.record ? "base" : record.locale!;
597
+ if (!expected[locale]) continue;
598
+ const output = outputStrings(locale === "base" ? data : { ...data, ...translations[locale] });
599
+ const { recall, missing } = textRecall(expected[locale], output, mapping.textMatch);
600
+ recalls.push(recall);
601
+ if (recall < minRecall)
602
+ ledger.issues.push({
603
+ kind: "low-recall",
604
+ key: record.key,
605
+ message: `only ${(recall * 100).toFixed(1)}% of source text reaches the document (missing e.g. ${missing.join(", ")})`,
606
+ });
607
+ }
608
+
609
+ if (collection.fields.slug?.type === "slug") {
610
+ for (const { record } of outputs) {
611
+ if (typeof record.permalink !== "string") continue;
612
+ const locale = record === base.record ? data._sourceLocale : record.locale;
613
+ if (locale !== data._sourceLocale && !translations[String(locale)]) continue;
614
+ const slugValue =
615
+ locale === data._sourceLocale ? data.slug : (translations[String(locale)]?.slug ?? data.slug);
616
+ const route = documentRoute(
617
+ collection,
618
+ { slug: slugify(String(slugValue ?? "")) },
619
+ (locale as string | undefined) ?? defaultLocale,
620
+ defaultLocale,
621
+ );
622
+ const before = normalizePath(record.permalink);
623
+ routes.add(route);
624
+ if (route !== before) {
625
+ ledger.issues.push({ kind: "url-change", key: record.key, message: `${before} → ${route}` });
626
+ urlChanges.push({ key: record.key, collection: collection.slug, docId, from: before, to: route });
627
+ }
628
+ }
629
+ }
630
+ docLedger.set(docId, ledger);
631
+
632
+ docs.push({
633
+ docId,
634
+ collection: collection.slug,
635
+ sourceKeys: outputs.map((o) => o.record.key),
636
+ data,
637
+ translations,
638
+ expected,
639
+ media: [...docMedia],
640
+ textMatch: mapping.textMatch ?? "multiset",
641
+ hash: sha({ data, translations }),
642
+ });
643
+ ledger.docs++;
644
+ }
645
+
646
+ for (const [field, stat] of divergence) {
647
+ const config = collection.fields[field] ?? { type: "text" };
648
+ const share = `${stat.differs} of ${stat.translations} translations`;
649
+ const where = stat.samples.length ? ` (e.g. ${stat.samples.join(", ")})` : "";
650
+ if (!translatable.has(field) && stat.differs) {
651
+ ledger.issues.push({
652
+ kind: "lost-translation",
653
+ message: isTranslatedText(config)
654
+ ? `"${field}" differs in ${share} but isn't translatable; those values are dropped${where}. Text: make it translatable.`
655
+ : `"${field}" differs in ${share} but isn't translatable; the original's value is kept${where}.${stat.differs / stat.translations >= 0.25 ? " That many differences may be intended: consider translatable." : ""}`,
656
+ });
657
+ } else if (
658
+ translatable.has(field) &&
659
+ !isTranslatedText(config) &&
660
+ // Terms exist per language; matching slugs don't make the field shared.
661
+ config.admin?.component !== "taxonomy-select" &&
662
+ stat.translations >= 3 &&
663
+ stat.differs / stat.translations < 0.1
664
+ ) {
665
+ ledger.issues.push({
666
+ kind: "translation-advice",
667
+ message: `"${field}" is translatable but differs in only ${share}${where}; likely shared: drop translatable and the original's value is kept.`,
668
+ });
669
+ }
670
+ }
671
+
672
+ for (const [field, keys] of inherits)
673
+ ledger.issues.push({
674
+ kind: "translation-advice",
675
+ message: `"${field}" is empty in ${keys.length} translation(s), which therefore show the original's value (e.g. ${keys.slice(0, 3).join(", ")}); fill them in Kide if the original's language is wrong there.`,
676
+ });
677
+
678
+ ledger.recall = {
679
+ min: recalls.length ? Math.min(...recalls) : 1,
680
+ avg: recalls.length ? recalls.reduce((a, b) => a + b, 0) / recalls.length : 1,
681
+ };
682
+ const usedDrops = new Set<string>();
683
+ for (const [p, stat] of [...domain.entries()].sort(([a], [b]) => a.localeCompare(b))) {
684
+ const drop = firstDrop(mapping, p);
685
+ if (drop) usedDrops.add(drop[0]);
686
+ const status = isCovered(p, reads) ? "mapped" : drop ? "dropped" : "unaccounted";
687
+ ledger.paths.push({ path: p, ...stat, status, reason: status === "dropped" ? drop![1] : undefined });
688
+ }
689
+ ledger.staleDrops = Object.keys(mapping.drop ?? {}).filter((d) => !usedDrops.has(d));
690
+ };
691
+ for (const p of prepared) mapPrepared(p);
692
+
693
+ // `url-change` mappings run last: their records are the URL changes the other mappings produced.
694
+ const deferred = active.filter((m) => m.source === URL_CHANGES);
695
+ if (deferred.length) {
696
+ const list: StagedRecord[] = urlChanges.map((c, i) => ({
697
+ key: `${URL_CHANGES}:${c.key}`,
698
+ type: URL_CHANGES,
699
+ id: i + 1,
700
+ status: "publish",
701
+ from: c.from,
702
+ to: c.to,
703
+ collection: c.collection,
704
+ docId: c.docId,
705
+ sourceKey: c.key,
706
+ }));
707
+ byType.set(URL_CHANGES, list);
708
+ for (const r of list) byKey.set(r.key, r);
709
+ const start = prepared.length;
710
+ for (const m of deferred) prepareMapping(m);
711
+ for (const p of prepared.slice(start)) mapPrepared(p);
712
+ }
713
+
714
+ // Internal links still carry WordPress URLs; point every one whose document moved at its Kide route
715
+ // (rich text, link fields, menus, redirect targets). A redirect's `from` is the old URL on purpose.
716
+ const moved = new Map(urlChanges.map((c) => [c.from, c.to]));
717
+ if (moved.size) {
718
+ const movedTo = (v: string) => {
719
+ const p = sitePath(v);
720
+ return p === null ? undefined : moved.get(normalizePath(p));
721
+ };
722
+ const relink = (value: unknown, key?: string): unknown => {
723
+ if (typeof value === "string") return key === "from" ? value : (movedTo(value) ?? value);
724
+ if (Array.isArray(value)) return value.map((v) => relink(v));
725
+ if (!value || typeof value !== "object") return value;
726
+ return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, relink(v, k)]));
727
+ };
728
+ for (const d of docs) {
729
+ d.data = relink(d.data) as Record<string, unknown>;
730
+ d.translations = relink(d.translations) as Record<string, Record<string, unknown>>;
731
+ d.hash = sha({ data: d.data, translations: d.translations });
732
+ }
733
+ }
734
+
735
+ // Redirects after the move: one pointing at itself, or away from a route a document now owns, is
736
+ // dropped; of two from the same URL, the site's own rule wins over a generated url-change one.
737
+ const isRedirect = (d: PlannedDoc) => {
738
+ const fields = collections.get(d.collection)?.fields;
739
+ return !!fields?.from && !!fields?.to;
740
+ };
741
+ const redirectPath = (v: unknown) => {
742
+ const p = typeof v === "string" ? sitePath(v) : null;
743
+ return p === null ? null : normalizePath(p);
744
+ };
745
+ const generated = (d: PlannedDoc) => d.sourceKeys[0]?.startsWith(`${URL_CHANGES}:`);
746
+ const seenFrom = new Map<string, PlannedDoc>();
747
+ const dropped = new Set<PlannedDoc>();
748
+ const drop = (d: PlannedDoc, message: string) => {
749
+ dropped.add(d);
750
+ const ledger = docLedger.get(d.docId);
751
+ if (ledger) {
752
+ ledger.docs--;
753
+ ledger.issues.push({ kind: "warning", key: d.sourceKeys[0], message });
754
+ }
755
+ };
756
+ for (const d of [...docs].sort((a, b) => Number(generated(a)) - Number(generated(b)))) {
757
+ if (!isRedirect(d)) continue;
758
+ const from = redirectPath(d.data.from);
759
+ const to = redirectPath(d.data.to);
760
+ if (!from) continue;
761
+ if (from === to) drop(d, `redirect ${from} points at itself once routes are flat — left out`);
762
+ else if (routes.has(from)) drop(d, `redirect from ${from} would hide the document now served there — left out`);
763
+ else if (seenFrom.has(from))
764
+ drop(d, `redirect from ${from} duplicates ${seenFrom.get(from)!.sourceKeys[0]} — left out`);
765
+ else seenFrom.set(from, d);
766
+ }
767
+ if (dropped.size) docs.splice(0, docs.length, ...docs.filter((d) => !dropped.has(d)));
768
+
769
+ // Ids are unique across collections: the plan and the applied state are keyed by id alone.
770
+ const ids = new Map<string, PlannedDoc[]>();
771
+ for (const d of docs) ids.set(d.docId, [...(ids.get(d.docId) ?? []), d]);
772
+ for (const [id, list] of ids)
773
+ if (list.length > 1)
774
+ docLedger.get(list[1].docId)?.issues.push({
775
+ kind: "duplicate-id",
776
+ message: `${list.length} documents share the id "${id}" (${list.map((d) => `${d.sourceKeys[0]} → ${d.collection}`).join(", ")}); give one mapping an \`id\``,
777
+ });
778
+
779
+ // Every taxonomy-select value must name a term of its taxonomy in that locale.
780
+ const termSets = new Map<string, Set<string>>();
781
+ const collectTerms = (terms: unknown, into: Set<string>) => {
782
+ for (const t of Array.isArray(terms) ? terms : []) {
783
+ if (t && typeof t === "object") {
784
+ if (typeof (t as any).slug === "string") into.add((t as any).slug);
785
+ collectTerms((t as any).children, into);
786
+ }
787
+ }
788
+ };
789
+ const flagBundles = (d: PlannedDoc, terms: unknown, locale: string) => {
790
+ for (const t of Array.isArray(terms) ? terms : []) {
791
+ if (!t || typeof t !== "object") continue;
792
+ if (termParts((t as any).name).length > 1)
793
+ docLedger.get(d.docId)?.issues.push({
794
+ kind: "bundled-term",
795
+ key: d.sourceKeys[0],
796
+ message: `"${(t as any).name}" (${locale}) is several terms in one; split it with termParts()`,
797
+ });
798
+ flagBundles(d, (t as any).children, locale);
799
+ }
800
+ };
801
+ for (const d of docs.filter((x) => x.collection === "taxonomies")) {
802
+ const base = String(d.data._sourceLocale ?? defaultLocale ?? "");
803
+ flagBundles(d, d.data.terms, base);
804
+ for (const [locale, overlay] of Object.entries(d.translations))
805
+ if (overlay.terms) flagBundles(d, overlay.terms, locale);
806
+ const set = new Set<string>();
807
+ collectTerms(d.data.terms, set);
808
+ termSets.set(`${d.data.slug}|${base}`, set);
809
+ for (const [locale, overlay] of Object.entries(d.translations)) {
810
+ const own = new Set<string>();
811
+ collectTerms(overlay.terms ?? d.data.terms, own);
812
+ termSets.set(`${d.data.slug}|${locale}`, own);
813
+ }
814
+ }
815
+ if (termSets.size) {
816
+ for (const d of docs) {
817
+ const collection = collections.get(d.collection)!;
818
+ for (const [name, field] of Object.entries(collection.fields)) {
819
+ if (field.admin?.component !== "taxonomy-select") continue;
820
+ const taxonomy = field.admin.taxonomy ?? field.admin.placeholder ?? "";
821
+ const check = (value: unknown, locale: string) => {
822
+ const set = termSets.get(`${taxonomy}|${locale}`);
823
+ for (const v of Array.isArray(value) ? value : value ? [value] : []) {
824
+ if (!set?.has(String(v)))
825
+ docLedger.get(d.docId)?.issues.push({
826
+ kind: "dangling-term",
827
+ key: d.sourceKeys[0],
828
+ message: `${name}: "${v}" isn't a term of taxonomy "${taxonomy}" (${locale})`,
829
+ });
830
+ }
831
+ };
832
+ check(d.data[name], String(d.data._sourceLocale ?? defaultLocale ?? ""));
833
+ for (const [locale, overlay] of Object.entries(d.translations))
834
+ if (name in overlay) check(overlay[name], locale);
835
+ }
836
+ }
837
+ }
838
+
839
+ // A drop list shared between mappings is only stale when it matches nothing in any of them.
840
+ const allPaths = ledgers.flatMap((l) => l.paths.map((p) => p.path));
841
+ const reported = new Set<string>();
842
+ for (const l of ledgers)
843
+ l.staleDrops = l.staleDrops.filter((d) => {
844
+ if (reported.has(d) || allPaths.some((p) => matchesPath(d, p))) return false;
845
+ return reported.add(d);
846
+ });
847
+
848
+ const mapped = new Set(mappings.flatMap((m) => [m.source, ...(isSkip(m) ? [] : (m.includes ?? []))]));
849
+ const unmapped = options.only
850
+ ? []
851
+ : [...byType.entries()]
852
+ .filter(([type]) => !mapped.has(type) && type !== "wp_block")
853
+ .map(([type, list]) => ({ type, records: list.length }));
854
+
855
+ const blocking: string[] = [...splitProblems];
856
+ for (const u of unmapped)
857
+ blocking.push(`source type \`${u.type}\` (${u.records} records) has no mapping or skipSource()`);
858
+ for (const l of ledgers) {
859
+ const open = l.paths.filter((p) => p.status === "unaccounted");
860
+ const label = shared.has(l.source) ? `${l.source} → ${l.collection}` : l.source;
861
+ if (open.length) blocking.push(`\`${label}\`: ${open.length} unaccounted path(s)`);
862
+ const counts = new Map<IssueKind, number>();
863
+ for (const i of l.issues) if (BLOCKING.includes(i.kind)) counts.set(i.kind, (counts.get(i.kind) ?? 0) + 1);
864
+ for (const [kind, n] of counts) blocking.push(`\`${label}\`: ${n} ${kind} issue(s)`);
865
+ }
866
+
867
+ return { ledgers, unmapped, docs, media, blocking, urlChanges, durationMs: Date.now() - started };
868
+ };
869
+
870
+ const hostOf = (u: string) => {
871
+ try {
872
+ return new URL(u).host;
873
+ } catch {
874
+ return "";
875
+ }
876
+ };
877
+
878
+ type ContextDeps = {
879
+ mapping: Mapping;
880
+ record: StagedRecord;
881
+ tracker: Tracker;
882
+ issues: Issue[];
883
+ docMedia: Set<string>;
884
+ docIdByKey: Map<string, string>;
885
+ collectionByKey: Map<string, string>;
886
+ byKey: Map<string, StagedRecord>;
887
+ media: Map<string, StagedMedia>;
888
+ fileIndex: Map<string, string>;
889
+ reusable: (ref: number) => string | undefined;
890
+ uploadsDir: string | undefined;
891
+ urlToMediaKey: (url: string, allowExternal: boolean) => string | null;
892
+ internalUrl: (url: string) => string;
893
+ recordForUrl: (url: string) => StagedRecord | undefined;
894
+ byType: Map<string, StagedRecord[]>;
895
+ byGroup: Map<string, StagedRecord[]>;
896
+ collection: CollectionConfig;
897
+ /** Block types the collection's content fields declare (decides how core blocks convert). */
898
+ declared: Record<string, Record<string, unknown>>;
899
+ /** Once-per-mapping notices already reported. */
900
+ notices: Set<string>;
901
+ };
902
+
903
+ const makeContext = (deps: ContextDeps): MapContext => {
904
+ const { mapping, record, tracker, issues, docMedia, docIdByKey, byKey, media } = deps;
905
+ const issue = (kind: IssueKind, message: string) => issues.push({ kind, key: record.key, message });
906
+
907
+ const claimMedia = (claimed: string) => {
908
+ // Polylang copies an attachment per language; the same file is one medium.
909
+ const file = media.get(claimed)?.file;
910
+ const key = (file && deps.fileIndex.get(file)) || claimed;
911
+ docMedia.add(key);
912
+ const m = media.get(key);
913
+ if (deps.uploadsDir && m?.file && !existsSync(path.join(deps.uploadsDir, m.file)))
914
+ issue("missing-media", `${m.file} not found in ${deps.uploadsDir}`);
915
+ return mediaToken(key);
916
+ };
917
+
918
+ const resolveRef = (prefix: "post" | "term", id: unknown, optional?: boolean): string | null => {
919
+ const raw = id && typeof id === "object" ? ((id as any).ID ?? (id as any).term_id ?? (id as any).id) : id;
920
+ if (raw == null || raw === "" || raw === 0 || raw === "0" || raw === false) return null;
921
+ // A full record key (`form:4`) addresses any staged source, not only posts and terms.
922
+ const key = typeof raw === "string" && /^[a-z_:-]+:\d+$/.test(raw) ? raw : `${prefix}:${Number(raw)}`;
923
+ const docId = docIdByKey.get(key);
924
+ if (docId) return docId;
925
+ if (!optional) {
926
+ const target = byKey.get(key);
927
+ issue(
928
+ "dangling-ref",
929
+ `${prefix} ${raw} ${target ? `(${target.type}, ${target.status}) isn't mapped` : "doesn't exist in the source"} — map it, or pass { optional: true }`,
930
+ );
931
+ }
932
+ return null;
933
+ };
934
+
935
+ const fileLinkKey = (url: string) =>
936
+ /\.(pdf|docx?|xlsx?|pptx?|zip|jpe?g|png|gif|webp|svg|mp4|mp3)$/i.test(url.split(/[?#]/)[0])
937
+ ? deps.urlToMediaKey(url, false)
938
+ : null;
939
+
940
+ const shortcodeHandlers = (html: string): Record<string, ShortcodeHandler> => {
941
+ const handlers: Record<string, ShortcodeHandler> = {};
942
+ for (const name of new Set(findShortcodes(stripComments(html)))) {
943
+ const own =
944
+ mapping.shortcodes?.[name] ??
945
+ (name === "gravityform" && deps.declared.form && !firstDrop(mapping, "shortcode:gravityform")
946
+ ? (a: Record<string, string>) => formBlock(a.id, a.title === "true", a.description === "true")
947
+ : undefined);
948
+ const fallback = mapping.shortcodes?.["*"];
949
+ if (own === "keep") tracker.read(`shortcode:${name}`, name);
950
+ else if (own) handlers[name] = own;
951
+ else if (firstDrop(mapping, `shortcode:${name}`)) handlers[name] = () => "";
952
+ else if (DEFAULT_SHORTCODES[name]) handlers[name] = DEFAULT_SHORTCODES[name];
953
+ else if (fallback === "keep") tracker.read(`shortcode:${name}`, name);
954
+ else if (fallback) handlers[name] = fallback;
955
+ }
956
+ return handlers;
957
+ };
958
+
959
+ const rewriteNodes = (nodes: RichTextNode[]): RichTextNode[] => {
960
+ for (const node of nodes) {
961
+ if (node.type === "image" && typeof node.src === "string") {
962
+ const key = deps.urlToMediaKey(node.src, false);
963
+ if (key) node.src = claimMedia(key);
964
+ }
965
+ if (typeof node.href === "string") {
966
+ const key = fileLinkKey(node.href);
967
+ node.href = key ? claimMedia(key) : deps.internalUrl(node.href);
968
+ }
969
+ if (Array.isArray(node.children)) rewriteNodes(node.children as RichTextNode[]);
970
+ }
971
+ return nodes;
972
+ };
973
+
974
+ // Shortcode handlers that return content nodes leave this token; `splice` swaps the nodes in.
975
+ const TOKEN = /⟦(\d+)⟧/g;
976
+ const splice = (nodes: ContentNode[], embedded: ContentNode[][]): ContentNode[] => {
977
+ const out: ContentNode[] = [];
978
+ const textOf = (n: any): string =>
979
+ typeof n.value === "string" ? n.value : (n.children ?? []).map(textOf).join("");
980
+ const strip = (n: any): any => {
981
+ if (typeof n.value === "string") return { ...n, value: n.value.replace(TOKEN, "") };
982
+ return Array.isArray(n.children) ? { ...n, children: n.children.map(strip) } : n;
983
+ };
984
+ for (const node of nodes) {
985
+ const found = [...JSON.stringify(node).matchAll(TOKEN)].map((m) => embedded[Number(m[1])] ?? []);
986
+ if (!found.length) {
987
+ out.push(node);
988
+ continue;
989
+ }
990
+ const rest = strip(node);
991
+ if (textOf(rest).trim()) out.push(rest);
992
+ out.push(...found.flat());
993
+ }
994
+ return out;
995
+ };
996
+
997
+ // `fromSource: false` for HTML a block handler built: its text was already expected via the reads that fed it.
998
+ const htmlNodes = (html: string, fromSource = true): ContentNode[] => {
999
+ const embedded: ContentNode[][] = [];
1000
+ const prepared = replaceShortcodes(
1001
+ html,
1002
+ shortcodeHandlers(html),
1003
+ (name) => tracker.read(`shortcode:${name}`, name),
1004
+ (nodes) => `⟦${embedded.push(nodes as ContentNode[]) - 1}⟧`,
1005
+ );
1006
+ const withParagraphs = /<p[\s>]/i.test(prepared) ? prepared : autop(prepared);
1007
+ if (fromSource) tracker.expect(htmlText(withParagraphs.replace(TOKEN, " ")));
1008
+ const nodes = rewriteNodes(htmlToRichText(withParagraphs).children as RichTextNode[]) as ContentNode[];
1009
+ return embedded.length ? splice(nodes, embedded) : nodes;
1010
+ };
1011
+
1012
+ const block = (blockType: string, fields: Record<string, unknown>): ContentNode => ({
1013
+ type: "block",
1014
+ blockType,
1015
+ fields,
1016
+ });
1017
+ const paragraph = (value: string, bold = false): ContentNode => ({
1018
+ type: "paragraph",
1019
+ children: [{ type: "text", value, ...(bold ? { bold: true } : {}) }],
1020
+ });
1021
+ const fallback = (name: string, blockType: string) => {
1022
+ if (deps.notices.has(name)) return;
1023
+ deps.notices.add(name);
1024
+ issue(
1025
+ "warning",
1026
+ `${name} became plain content; declare ${blockType === "form" ? "standardBlocks({ forms: true })" : `standardBlocks().${blockType}`} on the content field to keep it`,
1027
+ );
1028
+ };
1029
+ const caption = (html: string) => htmlText(/<figcaption[^>]*>([\s\S]*?)<\/figcaption>/i.exec(html)?.[1] ?? "");
1030
+
1031
+ // Code keeps its line breaks and indentation; rich text has no code node.
1032
+ const codeBlock = (b: WpBlock): ContentNode[] => {
1033
+ const html = blockHtml(b);
1034
+ if (!deps.declared.code) {
1035
+ fallback(b.name, "code");
1036
+ return htmlNodes(html);
1037
+ }
1038
+ tracker.expect(htmlText(html));
1039
+ const code = decodeEntities(html.replace(/<br\s*\/?>/gi, "\n").replace(/<[^>]+>/g, "")).replace(/^\n+|\s+$/g, "");
1040
+ const language = /language-([\w-]+)/.exec(html)?.[1] ?? String(b.attrs.language ?? "");
1041
+ return [block("code", { language, code })];
1042
+ };
1043
+
1044
+ const formBlock = (id: unknown, showTitle: boolean, showDescription: boolean): ContentNode[] => {
1045
+ const form = ctx.ref(`form:${Number(id)}`, { optional: true });
1046
+ if (!form) {
1047
+ issue("warning", `form ${String(id)} isn't imported — its block is left out`);
1048
+ return [];
1049
+ }
1050
+ return [block("form", { form, showTitle, showDescription })];
1051
+ };
1052
+
1053
+ /** Content nodes → items of a nested `blocks` field (accordion panel); undeclared blocks are hoisted after it. */
1054
+ const panelItems = (nodes: ContentNode[], types: Record<string, unknown>) => {
1055
+ const items: Record<string, unknown>[] = [];
1056
+ const hoisted: ContentNode[] = [];
1057
+ let prose: ContentNode[] = [];
1058
+ const flush = () => {
1059
+ if (prose.length) items.push({ type: "text", body: { type: "root", children: prose } });
1060
+ prose = [];
1061
+ };
1062
+ for (const node of nodes) {
1063
+ const b = node as { type: string; blockType?: string; fields?: Record<string, any> };
1064
+ if (b.type !== "block") prose.push(node);
1065
+ else if (b.blockType && b.blockType in types) {
1066
+ flush();
1067
+ items.push({ type: b.blockType, ...b.fields });
1068
+ } else if (b.blockType === "accordion") {
1069
+ prose.push({ type: "heading", level: 3, children: [{ type: "text", value: String(b.fields?.title ?? "") }] });
1070
+ flush();
1071
+ items.push(...((b.fields?.body as Record<string, unknown>[]) ?? []));
1072
+ } else hoisted.push(node);
1073
+ }
1074
+ flush();
1075
+ return { items, hoisted };
1076
+ };
1077
+
1078
+ /** Core blocks Kide converts itself: to `standardBlocks()` when the collection declares them, else to plain content. */
1079
+ const STANDARD: Record<string, (b: WpBlock) => ContentNode[]> = {
1080
+ "core/shortcode": (b) => htmlNodes(blockHtml(b)),
1081
+ "core/cover": (b) => {
1082
+ const src = ctx.media(b.attrs.id ?? b.attrs.url, { optional: true });
1083
+ const image: ContentNode[] = src ? [{ type: "image", src, alt: String(b.attrs.alt ?? "") }] : [];
1084
+ return [...image, ...renderBlocks(b.innerBlocks)];
1085
+ },
1086
+ "core/details": (b) => {
1087
+ const own = b.innerContent.filter((x): x is string => x !== null).join("");
1088
+ const title = htmlText(/<summary[^>]*>([\s\S]*?)<\/summary>/i.exec(own)?.[1] ?? "");
1089
+ tracker.expect(title);
1090
+ const inner = b.innerBlocks.length
1091
+ ? renderBlocks(b.innerBlocks)
1092
+ : htmlNodes(blockHtml(b).replace(/<summary[\s\S]*?<\/summary>/i, ""));
1093
+ const panel = deps.declared.accordion?.body as { type?: string; types?: Record<string, unknown> } | undefined;
1094
+ if (panel?.type !== "blocks") {
1095
+ fallback("core/details", "accordion");
1096
+ return [paragraph(title, true), ...inner];
1097
+ }
1098
+ const { items, hoisted } = panelItems(inner, panel.types ?? {});
1099
+ if (hoisted.length)
1100
+ issue(
1101
+ "warning",
1102
+ `accordion "${truncate(title, 40)}": ${hoisted.map((n) => (n as any).blockType).join(", ")} can't sit in a panel, placed after it; add it with standardBlocks({ panelBlocks })`,
1103
+ );
1104
+ return [block("accordion", { title, body: items }), ...hoisted];
1105
+ },
1106
+ "core/table": (b) => {
1107
+ const html = blockHtml(b);
1108
+ tracker.expect(htmlText(html));
1109
+ const rows = [...html.matchAll(/<tr[\s>][\s\S]*?<\/tr>/gi)].map((m) =>
1110
+ [...m[0].matchAll(/<t([dh])[^>]*>([\s\S]*?)<\/t\1>/gi)].map((c) => htmlText(c[2])).join(" | "),
1111
+ );
1112
+ if (deps.declared.table)
1113
+ return [
1114
+ block("table", {
1115
+ caption: caption(html),
1116
+ headerRow: /<thead/i.test(html),
1117
+ rows: rows.map((cells) => ({ cells })),
1118
+ }),
1119
+ ];
1120
+ fallback("core/table", "table");
1121
+ return [...rows.map((r) => paragraph(r)), ...(caption(html) ? [paragraph(caption(html))] : [])];
1122
+ },
1123
+ "core/video": (b) => {
1124
+ const html = blockHtml(b);
1125
+ const src = /<video[^>]*\ssrc="([^"]+)"/i.exec(html)?.[1] ?? "";
1126
+ const text = caption(html);
1127
+ tracker.expect(text);
1128
+ if (!deps.declared.video) {
1129
+ fallback("core/video", "video");
1130
+ return htmlNodes(`<p><a href="${src}">${text || src.split("/").pop()}</a></p>`, false);
1131
+ }
1132
+ const file = ctx.media(b.attrs.id, { optional: true }) ?? (src ? ctx.media(src) : null);
1133
+ return [block("video", { file, autoplay: /<video[^>]*\sautoplay/i.test(html), caption: text })];
1134
+ },
1135
+ "core/embed": (b) => {
1136
+ const url = String(b.attrs.url ?? "");
1137
+ tracker.expect(caption(blockHtml(b)));
1138
+ if (deps.declared.embed) return [block("embed", { url })];
1139
+ fallback("core/embed", "embed");
1140
+ return htmlNodes(`<p><a href="${url}">${url}</a></p>`, false);
1141
+ },
1142
+ "core/audio": (b) => {
1143
+ const html = blockHtml(b);
1144
+ const src = /<audio[^>]*\ssrc="([^"]+)"/i.exec(html)?.[1] ?? "";
1145
+ const text = caption(html);
1146
+ tracker.expect(text);
1147
+ if (!deps.declared.audio) {
1148
+ fallback("core/audio", "audio");
1149
+ return htmlNodes(`<p><a href="${src}">${text || src.split("/").pop()}</a></p>`, false);
1150
+ }
1151
+ return [
1152
+ block("audio", {
1153
+ file: ctx.media(b.attrs.id, { optional: true }) ?? (src ? ctx.media(src) : null),
1154
+ caption: text,
1155
+ }),
1156
+ ];
1157
+ },
1158
+ "core/code": (b) => codeBlock(b),
1159
+ "core/preformatted": (b) => codeBlock(b),
1160
+ "core/html": (b) => {
1161
+ const code = blockHtml(b);
1162
+ if (!deps.declared.html) {
1163
+ fallback("core/html", "html");
1164
+ return htmlNodes(code);
1165
+ }
1166
+ tracker.expect(htmlText(code));
1167
+ return [block("html", { code: code.trim() })];
1168
+ },
1169
+ "core/media-text": (b) => {
1170
+ const src = ctx.media(b.attrs.mediaId ?? b.attrs.mediaLink, { optional: true });
1171
+ return [
1172
+ ...(src ? [{ type: "image", src, alt: String(b.attrs.mediaAlt ?? "") } as ContentNode] : []),
1173
+ ...renderBlocks(b.innerBlocks),
1174
+ ];
1175
+ },
1176
+ "gravityforms/form": (b) => {
1177
+ if (deps.declared.form) return formBlock(b.attrs.formId, b.attrs.title !== false, b.attrs.description !== false);
1178
+ fallback("gravityforms/form", "form");
1179
+ return [];
1180
+ },
1181
+ };
1182
+
1183
+ /** A handler's view of a block; `done()` settles which of its text the handler consumed. */
1184
+ const blockInput = (b: WpBlock): { input: BlockInput; done: () => void } => {
1185
+ const base = `block:${b.name}`;
1186
+ tracker.read(base, b.name);
1187
+ let htmlRead = false;
1188
+ let innerUsed = false;
1189
+ let rendered: ContentNode[] | undefined;
1190
+ const children: Array<{ done: () => void }> = [];
1191
+ const mark = tracker.expected.length;
1192
+ const input: BlockInput = {
1193
+ name: b.name,
1194
+ attrs: track(Object.fromEntries(Object.entries(b.attrs).filter(([k]) => k !== "data")), `${base}.attrs`, tracker),
1195
+ data: track(blockData(b), `${base}.data`, tracker),
1196
+ get innerHTML() {
1197
+ htmlRead = true;
1198
+ return blockHtml(b);
1199
+ },
1200
+ inner: () => {
1201
+ innerUsed = true;
1202
+ return (rendered ??= renderBlocks(b.innerBlocks));
1203
+ },
1204
+ auto: (blockType = blockTypeOf(b.name)) => {
1205
+ const fields = deps.declared[blockType] as Record<string, FieldConfig> | undefined;
1206
+ if (!fields) throw new Error(`b.auto(): no declared block "${blockType}" on this collection's content fields`);
1207
+ return autoFields(fields, isAcfBlock(b) ? input.data : input.attrs, ctx);
1208
+ },
1209
+ children: () => {
1210
+ innerUsed = true;
1211
+ return b.innerBlocks.map((child) => {
1212
+ const view = blockInput(child);
1213
+ children.push(view);
1214
+ return view.input;
1215
+ });
1216
+ },
1217
+ };
1218
+ const done = () => {
1219
+ for (const child of children.splice(0)) child.done();
1220
+ // Inner blocks the handler never looked at are still source content: if it drops them, recall says so.
1221
+ if (!htmlRead && !innerUsed) for (const c of b.innerBlocks) tracker.expect(htmlText(blockHtml(c)));
1222
+ // Text of inner blocks the handler rendered or read is accounted there; count only the block's own markup.
1223
+ if (!htmlRead) return;
1224
+ const html = innerUsed ? b.innerContent.filter((x) => x !== null).join("") : blockHtml(b);
1225
+ // Pieces of that markup the handler passed to ctx.text/richText are part of it: count them once.
1226
+ const whole = words(htmlText(html)).join(" ");
1227
+ for (let i = tracker.expected.length - 1; i >= mark; i--)
1228
+ if (whole.includes(words(tracker.expected[i]).join(" "))) tracker.expected.splice(i, 1);
1229
+ tracker.read(`${base}.innerHTML`, html);
1230
+ };
1231
+ return { input, done };
1232
+ };
1233
+
1234
+ const blockSource = (b: WpBlock) => {
1235
+ const base = `block:${b.name}`;
1236
+ tracker.read(base, b.name);
1237
+ return isAcfBlock(b)
1238
+ ? track(blockData(b), `${base}.data`, tracker)
1239
+ : track(Object.fromEntries(Object.entries(b.attrs).filter(([k]) => k !== "data")), `${base}.attrs`, tracker);
1240
+ };
1241
+
1242
+ /**
1243
+ * A block filled by name from its own data, plus what it holds: an unfilled repeater takes one row
1244
+ * per child block (a carousel's cards), an unfilled `heading`/`title` its heading child's text.
1245
+ */
1246
+ const autoBlock = (b: WpBlock, fields: Record<string, FieldConfig>): ContentNode => {
1247
+ const values = autoFields(fields, blockSource(b), ctx);
1248
+ const children = b.innerBlocks.filter((c) => !CONVERTED.has(c.name));
1249
+ let usedHeading: WpBlock | undefined;
1250
+ for (const [name, field] of Object.entries(fields)) {
1251
+ if (name in values) continue;
1252
+ if (field.type === "json" && field.itemFields && children.length) {
1253
+ values[name] = children
1254
+ .map((c) => autoFields(field.itemFields!, blockSource(c), ctx))
1255
+ .filter((row) => Object.keys(row).length);
1256
+ } else if (field.type === "text" && (name === "heading" || name === "title")) {
1257
+ const heading = [...walkBlocks(b.innerBlocks)].find((c) => c.name === "core/heading");
1258
+ if (heading) {
1259
+ usedHeading = heading;
1260
+ tracker.read(`block:${heading.name}`, heading.name);
1261
+ for (const c of walkBlocks(b.innerBlocks))
1262
+ if (CONTAINER_BLOCKS.has(c.name)) tracker.read(`block:${c.name}`, c.name);
1263
+ values[name] = htmlText(blockHtml(heading));
1264
+ tracker.expect(values[name] as string);
1265
+ }
1266
+ }
1267
+ }
1268
+ // Content children that filled nothing (a paragraph, an unused heading) must still arrive somewhere.
1269
+ const content = (list: WpBlock[]): WpBlock[] =>
1270
+ list.flatMap((c) => (CONTAINER_BLOCKS.has(c.name) ? content(c.innerBlocks) : HTML_BLOCKS.has(c.name) ? [c] : []));
1271
+ for (const c of content(b.innerBlocks)) if (c !== usedHeading) tracker.expect(htmlText(blockHtml(c)));
1272
+ // The block's own saved markup is a render of its attributes; only words it adds must still arrive.
1273
+ const own = htmlText(b.innerContent.filter((x): x is string => x !== null).join(""));
1274
+ if (own) {
1275
+ tracker.reads.add(`block:${b.name}.innerHTML`);
1276
+ const known = new Set(words(JSON.stringify(b.attrs)));
1277
+ const extra = words(own).filter((w) => !known.has(w));
1278
+ if (extra.length) tracker.expect(extra.join(" "));
1279
+ }
1280
+ return block(blockTypeOf(b.name), values);
1281
+ };
1282
+
1283
+ const renderBlocks = (blocks: WpBlock[]): ContentNode[] => {
1284
+ const out: ContentNode[] = [];
1285
+ for (const b of blocks) {
1286
+ const base = `block:${b.name}`;
1287
+ const before = out.length;
1288
+ let converted = false;
1289
+ if (b.name === "core/block") {
1290
+ issue("warning", `reusable block ${String(b.attrs.ref)} not found in the source — skipped`);
1291
+ continue;
1292
+ }
1293
+ const handler = mapping.blocks?.[b.name];
1294
+ if (handler) {
1295
+ const { input, done } = blockInput(b);
1296
+ const result = handler(input, ctx);
1297
+ done();
1298
+ if (typeof result === "string") out.push(...htmlNodes(result, false));
1299
+ else if (Array.isArray(result)) out.push(...result);
1300
+ else if (result) out.push(result);
1301
+ } else if (firstDrop(mapping, base)) {
1302
+ continue;
1303
+ } else if (!CONVERTED.has(b.name) && !STANDARD[b.name] && deps.declared[blockTypeOf(b.name)]) {
1304
+ // A plugin, ACF or unconverted core block maps onto the declared block of the same name,
1305
+ // field by field, from its ACF data or its attributes.
1306
+ out.push(autoBlock(b, deps.declared[blockTypeOf(b.name)] as Record<string, FieldConfig>));
1307
+ } else if (STANDARD[b.name] ?? (b.name.startsWith("core-embed/") && STANDARD["core/embed"])) {
1308
+ tracker.read(base, b.name);
1309
+ // The built-in conversion decides what of a plugin block's attributes matters; all are accounted.
1310
+ JSON.stringify(track(b.attrs, `${base}.attrs`, new Tracker(tracker.reads)));
1311
+ tracker.reads.add(`${base}.innerHTML`);
1312
+ out.push(...(STANDARD[b.name] ?? STANDARD["core/embed"])(b));
1313
+ converted = true;
1314
+ } else if (CONTAINER_BLOCKS.has(b.name)) {
1315
+ tracker.read(base, b.name);
1316
+ out.push(...(b.innerBlocks.length ? renderBlocks(b.innerBlocks) : htmlNodes(b.innerHTML)));
1317
+ } else if (EMPTY_BLOCKS.has(b.name)) {
1318
+ tracker.read(base, b.name);
1319
+ } else if (HTML_BLOCKS.has(b.name)) {
1320
+ const markInner = (list: WpBlock[]) => {
1321
+ for (const inner of list) {
1322
+ tracker.read(`block:${inner.name}`, inner.name);
1323
+ markInner(inner.innerBlocks);
1324
+ }
1325
+ };
1326
+ tracker.read(base, b.name);
1327
+ markInner(b.innerBlocks);
1328
+ out.push(...htmlNodes(blockHtml(b)));
1329
+ converted = true;
1330
+ }
1331
+ // Anything else is unaccounted: left out of the document, listed in the ledger until handled or dropped.
1332
+ // Markup without text (a widget, a custom element) converts to nothing: keep it as HTML or say so.
1333
+ if (converted && out.length === before && MARKUP.test(blockHtml(b))) {
1334
+ if (deps.declared.html) out.push(block("html", { code: blockHtml(b).trim() }));
1335
+ else
1336
+ issue(
1337
+ "warning",
1338
+ `${b.name} holds markup but no text (${truncate(blockHtml(b), 50)}) — left out; declare standardBlocks().html to keep it`,
1339
+ );
1340
+ }
1341
+ }
1342
+ return out;
1343
+ };
1344
+
1345
+ const ctx: MapContext = {
1346
+ locale: record.locale,
1347
+ ref: (id, o) => resolveRef("post", id, o?.optional),
1348
+ refs: (ids, o) => {
1349
+ const list = Array.isArray(ids) ? ids : ids == null || ids === "" ? [] : [ids];
1350
+ return [...new Set(list.map((id) => resolveRef("post", id, o?.optional)).filter((x): x is string => !!x))];
1351
+ },
1352
+ term: (id, o) => resolveRef("term", id, o?.optional),
1353
+ refLink: (id, o) => {
1354
+ const docId = resolveRef("post", id, o?.optional);
1355
+ const raw = id && typeof id === "object" ? (id as any).ID : id;
1356
+ const target = byKey.get(`post:${Number(raw)}`);
1357
+ const collection = target && deps.collectionByKey.get(target.key);
1358
+ return docId && collection
1359
+ ? {
1360
+ type: "reference",
1361
+ docId,
1362
+ collection,
1363
+ url: typeof target.permalink === "string" ? normalizePath(target.permalink) : "",
1364
+ ...(target.title ? { title: String(target.title) } : {}),
1365
+ }
1366
+ : null;
1367
+ },
1368
+ media: (value, o) => {
1369
+ const raw =
1370
+ value && typeof value === "object" ? ((value as any).ID ?? (value as any).id ?? (value as any).url) : value;
1371
+ if (raw == null || raw === "" || raw === 0 || raw === "0" || raw === false) return null;
1372
+ // A snapshot of a since-deleted attachment still names its file.
1373
+ const url = value && typeof value === "object" ? (value as any).url : undefined;
1374
+ const key =
1375
+ typeof raw === "number" || /^\d+$/.test(String(raw))
1376
+ ? media.has(`att:${raw}`)
1377
+ ? `att:${raw}`
1378
+ : typeof url === "string" && url
1379
+ ? deps.urlToMediaKey(url, true)
1380
+ : null
1381
+ : deps.urlToMediaKey(String(raw), true);
1382
+ if (!key) {
1383
+ if (!o?.optional) issue("missing-media", `media ${truncate(raw, 60)} isn't in the source`);
1384
+ return null;
1385
+ }
1386
+ return claimMedia(key);
1387
+ },
1388
+ content: (html) => {
1389
+ tracker.consume(html);
1390
+ if (typeof html !== "string" || !html.trim()) return { type: "root", children: [] };
1391
+ // "keep" shortcodes are literal text wherever they sit, including blocks that never become rich text.
1392
+ for (const name of new Set(findShortcodes(stripComments(html)))) {
1393
+ const rule = mapping.shortcodes?.[name] ?? (DEFAULT_SHORTCODES[name] ? undefined : mapping.shortcodes?.["*"]);
1394
+ if (rule === "keep" && !firstDrop(mapping, `shortcode:${name}`)) tracker.read(`shortcode:${name}`, name);
1395
+ }
1396
+ return { type: "root", children: renderBlocks(expandBlocks(html, deps.reusable)) };
1397
+ },
1398
+ richText: (html) => {
1399
+ tracker.consume(html);
1400
+ if (typeof html !== "string" || !html.trim()) return { type: "root", children: [] };
1401
+ return { type: "root", children: htmlNodes(html) as RichTextNode[] };
1402
+ },
1403
+ text: (html) => {
1404
+ tracker.consume(html);
1405
+ const t = typeof html === "string" ? htmlText(html) : "";
1406
+ tracker.expect(t);
1407
+ return t;
1408
+ },
1409
+ link: (value) => {
1410
+ if (!value) return null;
1411
+ const v =
1412
+ typeof value === "string" ? { url: value } : (value as { url?: string; title?: string; target?: string });
1413
+ if (!v.url) return null;
1414
+ const key = fileLinkKey(v.url);
1415
+ const target = key ? undefined : deps.recordForUrl(v.url);
1416
+ const docId = target && docIdByKey.get(target.key);
1417
+ const collection = target && deps.collectionByKey.get(target.key);
1418
+ if (target && docId && collection)
1419
+ return {
1420
+ type: "reference",
1421
+ docId,
1422
+ collection,
1423
+ url: normalizePath(String(target.permalink)),
1424
+ ...(target.title ? { title: String(target.title) } : {}),
1425
+ ...(v.title ? { label: v.title } : {}),
1426
+ ...(v.target === "_blank" ? { newTab: true } : {}),
1427
+ };
1428
+ return {
1429
+ type: "custom",
1430
+ url: key ? claimMedia(key) : deps.internalUrl(v.url),
1431
+ ...(v.title ? { label: v.title } : {}),
1432
+ ...(v.target === "_blank" ? { newTab: true } : {}),
1433
+ };
1434
+ },
1435
+ date: (value) => {
1436
+ if (value == null || value === "") return null;
1437
+ const s = String(value);
1438
+ if (/^\d{8}$/.test(s)) return `${s.slice(0, 4)}-${s.slice(4, 6)}-${s.slice(6, 8)}`;
1439
+ if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(s)) return s.replace(" ", "T");
1440
+ if (!Number.isNaN(Date.parse(s))) return s;
1441
+ issue("warning", `unparseable date ${truncate(s, 40)}`);
1442
+ return null;
1443
+ },
1444
+ record: (keyOrId) =>
1445
+ byKey.get(typeof keyOrId === "number" || /^\d+$/.test(String(keyOrId)) ? `post:${keyOrId}` : String(keyOrId)),
1446
+ translation: (keyOrRecord, locale) => {
1447
+ const raw = keyOrRecord && typeof keyOrRecord === "object" ? (keyOrRecord as any).key : keyOrRecord;
1448
+ const r = byKey.get(String(raw));
1449
+ if (!r) return undefined;
1450
+ if (r.locale === locale) return r as SourceRecord;
1451
+ return deps.byGroup.get(String(r.group))?.find((m) => m.locale === locale) as SourceRecord | undefined;
1452
+ },
1453
+ records: (type, filter) => {
1454
+ if (!mapping.includes?.includes(type))
1455
+ throw new Error(`ctx.records("${type}") needs "${type}" in the mapping's includes`);
1456
+ return (deps.byType.get(type) ?? [])
1457
+ .filter((r) => !filter || filter(r))
1458
+ .map((r) => track(r, `${type}:`, tracker) as SourceRecord);
1459
+ },
1460
+ auto: (src) => autoRecord(src, deps.collection, ctx),
1461
+ menuItems: (menu) => menuItems(menu, ctx),
1462
+ termTree: (taxonomy, options) => termTree(taxonomy, ctx, options),
1463
+ form: (form) => gravityForm(form, ctx),
1464
+ warn: (message) => issue("warning", message),
1465
+ };
1466
+ return ctx;
1467
+ };