@kidecms/wordpress 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/bin/kide-wp.mjs +6 -0
- package/package.json +37 -3
- package/src/apply.ts +332 -0
- package/src/automap.ts +143 -0
- package/src/cli.ts +221 -0
- package/src/domain.ts +268 -0
- package/src/gutenberg.ts +125 -0
- package/src/index.ts +7 -0
- package/src/live.ts +282 -0
- package/src/mapping.ts +170 -0
- package/src/mysql-dump.ts +202 -0
- package/src/php.ts +62 -0
- package/src/plan.ts +1467 -0
- package/src/profile.ts +123 -0
- package/src/report.ts +128 -0
- package/src/staging.ts +77 -0
- package/src/structures.ts +199 -0
- package/src/terms.ts +6 -0
- package/src/verify.ts +156 -0
- package/src/wordpress.ts +472 -0
package/src/cli.ts
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mapping-driven content import (WordPress for now).
|
|
3
|
+
*
|
|
4
|
+
* kide-wp scout <dump.sql[.gz]> [--uploads <wp-content/uploads dir>] [--media-base <url>] [--site-url <url>] [--prefix wp_]
|
|
5
|
+
* kide-wp sample <type> [--id <n>] [--n 2] [--full]
|
|
6
|
+
* kide-wp plan [--only <type,type>] [--show <id|key>] [--min-recall 0.95]
|
|
7
|
+
* kide-wp apply [--all] [--force] [--no-prune] [--concurrency 12]
|
|
8
|
+
* kide-wp verify [--min-recall 0.95]
|
|
9
|
+
* kide-wp check-live [--base <url>] [--min-score 0.9] [--concurrency 6] [--limit N] [--refresh]
|
|
10
|
+
*
|
|
11
|
+
* Mappings live in migration/*.map.ts. Outputs go to .kide/import/.
|
|
12
|
+
*/
|
|
13
|
+
import { readdirSync, writeFileSync } from "node:fs";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
|
|
16
|
+
import { applyImport } from "./apply";
|
|
17
|
+
import { checkLive, liveScore, renderLive } from "./live";
|
|
18
|
+
import { truncate } from "./domain";
|
|
19
|
+
import { loadMappings, planImport } from "./plan";
|
|
20
|
+
import { buildProfile, renderProfile } from "./profile";
|
|
21
|
+
import { renderLedger, savePlan } from "./report";
|
|
22
|
+
import { getMeta, importDir, loadMedia, loadRecords, openStaging } from "./staging";
|
|
23
|
+
import { verifyImport } from "./verify";
|
|
24
|
+
import { scoutWordPress } from "./wordpress";
|
|
25
|
+
import { loadProjectConfig, loadProjectRuntime } from "@kidecms/core/internals/project.ts";
|
|
26
|
+
|
|
27
|
+
// Per-document audit lines would bury the report on a real site.
|
|
28
|
+
process.env.CMS_LOG_LEVEL ??= "warn";
|
|
29
|
+
|
|
30
|
+
const USAGE = `Usage: kide-wp <command> (playbook: MIGRATING.md)
|
|
31
|
+
|
|
32
|
+
scout <dump.sql[.gz]> [--uploads <dir>] [--site-url <url>] [--media-base <url>] [--prefix wp_]
|
|
33
|
+
sample <type> [--id <n>] [--n 2] [--full] staged records of a source type
|
|
34
|
+
plan [--only <type,…>] [--show <id|key>] [--min-recall 0.95]
|
|
35
|
+
apply [--all] [--force] [--no-prune] [--concurrency 12]
|
|
36
|
+
verify [--min-recall 0.95]
|
|
37
|
+
check-live [--base <url>] [--min-score 0.9] [--concurrency 6] [--limit N] [--refresh]
|
|
38
|
+
|
|
39
|
+
Mappings: migration/*.map.ts · reports: .kide/import/{profile,ledger,verify,live}.md`;
|
|
40
|
+
|
|
41
|
+
const [command, ...args] = process.argv.slice(2);
|
|
42
|
+
const flag = (name: string) => args.includes(`--${name}`);
|
|
43
|
+
// Every subcommand writes or fetches something, so asking for help must never run one.
|
|
44
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
45
|
+
console.log(USAGE);
|
|
46
|
+
process.exit(0);
|
|
47
|
+
}
|
|
48
|
+
const option = (name: string) => {
|
|
49
|
+
const i = args.indexOf(`--${name}`);
|
|
50
|
+
return i >= 0 ? args[i + 1] : undefined;
|
|
51
|
+
};
|
|
52
|
+
const positional = args.filter((a, i) => !a.startsWith("--") && !args[i - 1]?.startsWith("--"));
|
|
53
|
+
const out = (file: string) => path.relative(process.cwd(), path.join(importDir(), file));
|
|
54
|
+
const fail = (message: string): never => {
|
|
55
|
+
console.error(`[kide-wp] ${message}`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const staging = openStaging();
|
|
60
|
+
|
|
61
|
+
const shorten = (value: unknown, full: boolean): unknown => {
|
|
62
|
+
if (full) return value;
|
|
63
|
+
if (typeof value === "string") return value.length > 400 ? `${value.slice(0, 400)}… (${value.length} chars)` : value;
|
|
64
|
+
if (Array.isArray(value)) return value.map((v) => shorten(v, full));
|
|
65
|
+
if (value && typeof value === "object")
|
|
66
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, shorten(v, full)]));
|
|
67
|
+
return value;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
switch (command) {
|
|
72
|
+
case "scout": {
|
|
73
|
+
const dump = positional[0] ?? fail("usage: kide-wp scout <dump.sql[.gz]> [--uploads <dir>]");
|
|
74
|
+
const started = Date.now();
|
|
75
|
+
let uploads = option("uploads") ? path.resolve(option("uploads")!) : undefined;
|
|
76
|
+
// A Bedrock checkout has an empty uploads folder (just .gitkeep): download from the site instead.
|
|
77
|
+
if (
|
|
78
|
+
uploads &&
|
|
79
|
+
!readdirSync(uploads, { recursive: true }).some((f) => !String(f).split("/").pop()!.startsWith("."))
|
|
80
|
+
) {
|
|
81
|
+
console.log(`[kide-wp] ${uploads} holds no files — media will be downloaded from the site on apply.`);
|
|
82
|
+
uploads = undefined;
|
|
83
|
+
}
|
|
84
|
+
const summary = await scoutWordPress(staging, path.resolve(dump), {
|
|
85
|
+
prefix: option("prefix"),
|
|
86
|
+
uploadsDir: uploads,
|
|
87
|
+
mediaBaseUrl: option("media-base"),
|
|
88
|
+
siteUrl: option("site-url"),
|
|
89
|
+
});
|
|
90
|
+
const meta = Object.fromEntries(
|
|
91
|
+
["source", "siteUrl", "prefix", "uploadsUrl", "uploadsDir", "scoutedAt"].map((k) => [k, getMeta(staging, k)]),
|
|
92
|
+
);
|
|
93
|
+
const records = loadRecords(staging);
|
|
94
|
+
const profile = buildProfile(records);
|
|
95
|
+
writeFileSync(path.join(importDir(), "profile.md"), renderProfile(meta, profile, loadMedia(staging)));
|
|
96
|
+
writeFileSync(path.join(importDir(), "profile.json"), JSON.stringify({ meta, types: profile }, null, 2));
|
|
97
|
+
console.log(
|
|
98
|
+
`[kide-wp] Staged ${summary.records} records + ${summary.media} media from ${summary.siteUrl || dump} (prefix ${summary.prefix}) in ${Date.now() - started}ms.`,
|
|
99
|
+
);
|
|
100
|
+
for (const [type, n] of Object.entries(summary.types).sort((a, b) => b[1] - a[1]))
|
|
101
|
+
console.log(` ${type.padEnd(28)} ${n}`);
|
|
102
|
+
console.log(`Read ${out("profile.md")}, then write migration/<type>.map.ts and run \`kide-wp plan\`.`);
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
case "sample": {
|
|
107
|
+
const type = positional[0] ?? fail("usage: kide-wp sample <type> [--id <n>] [--n 2] [--full]");
|
|
108
|
+
const id = option("id");
|
|
109
|
+
const n = Number(option("n") ?? 2);
|
|
110
|
+
const list = loadRecords(staging).filter((r) => r.type === type && (!id || String(r.id) === id));
|
|
111
|
+
if (!list.length) fail(`no staged records for ${type}${id ? ` with id ${id}` : ""}`);
|
|
112
|
+
console.log(JSON.stringify(shorten(list.slice(0, id ? 1 : n), flag("full")), null, 2));
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
case "plan": {
|
|
117
|
+
await loadProjectRuntime();
|
|
118
|
+
const config = await loadProjectConfig();
|
|
119
|
+
const only = option("only")?.split(",");
|
|
120
|
+
const plan = await planImport(staging, config, await loadMappings(), {
|
|
121
|
+
minRecall: option("min-recall") ? Number(option("min-recall")) : undefined,
|
|
122
|
+
only,
|
|
123
|
+
});
|
|
124
|
+
const show = option("show");
|
|
125
|
+
if (show) {
|
|
126
|
+
const key = /^\d+$/.test(show) ? `post:${show}` : show;
|
|
127
|
+
const doc = plan.docs.find((d) => d.sourceKeys.includes(key) || d.docId === show);
|
|
128
|
+
const issues = plan.ledgers.flatMap((l) => l.issues.filter((i) => i.key === key));
|
|
129
|
+
console.log(JSON.stringify({ doc: doc && { ...doc, expected: undefined }, issues }, null, 2));
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
writeFileSync(path.join(importDir(), "ledger.md"), renderLedger(plan));
|
|
133
|
+
if (!only) savePlan(staging, plan);
|
|
134
|
+
const totals = plan.ledgers.map((l) => {
|
|
135
|
+
if (l.skip) return ` ${l.source.padEnd(24)} skipped — ${truncate(l.skip, 60)}`;
|
|
136
|
+
const open = l.paths.filter((p) => p.status === "unaccounted").length;
|
|
137
|
+
const issues = l.issues.filter((i) => i.kind !== "warning").length;
|
|
138
|
+
return ` ${l.source.padEnd(24)} → ${String(l.collection).padEnd(16)} ${String(l.docs).padStart(5)} docs · ${open} unaccounted · ${issues} issues · recall min ${(l.recall.min * 100).toFixed(1)}%`;
|
|
139
|
+
});
|
|
140
|
+
console.log(`[kide-wp] Planned ${plan.docs.length} documents in ${plan.durationMs}ms.`);
|
|
141
|
+
console.log(totals.join("\n"));
|
|
142
|
+
if (plan.blocking.length) {
|
|
143
|
+
console.log(`\nBLOCKED (${plan.blocking.length}) — details in ${out("ledger.md")}:`);
|
|
144
|
+
for (const b of plan.blocking.slice(0, 30)) console.log(` - ${b}`);
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
} else {
|
|
147
|
+
console.log(
|
|
148
|
+
`\nREADY${only ? " (partial plan, not saved — run without --only to apply)" : ""}. Ledger: ${out("ledger.md")}`,
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
case "apply": {
|
|
155
|
+
const r = await applyImport(staging, {
|
|
156
|
+
force: flag("force"),
|
|
157
|
+
all: flag("all"),
|
|
158
|
+
prune: !flag("no-prune"),
|
|
159
|
+
concurrency: option("concurrency") ? Number(option("concurrency")) : undefined,
|
|
160
|
+
});
|
|
161
|
+
console.log(
|
|
162
|
+
`[kide-wp] ${r.created} created · ${r.updated} updated · ${r.unchanged} unchanged · ${r.deleted} deleted · media ${r.media.uploaded} uploaded, ${r.media.reused} reused · ${r.durationMs}ms`,
|
|
163
|
+
);
|
|
164
|
+
if (r.media.missing.length)
|
|
165
|
+
console.log(` ${r.media.missing.length} file(s) missing at the source (4xx), their references dropped:`);
|
|
166
|
+
for (const f of r.media.missing.slice(0, 20)) console.log(` ${f.key} — ${f.message}`);
|
|
167
|
+
for (const f of r.media.failed.slice(0, 20)) console.log(` media failed: ${f.key} — ${f.message}`);
|
|
168
|
+
for (const f of r.failed.slice(0, 20)) console.log(` failed: ${f.docId} — ${f.message}`);
|
|
169
|
+
if (r.failed.length || r.media.failed.length) process.exitCode = 1;
|
|
170
|
+
else console.log("Run `kide-wp verify` to reconcile the CMS against the plan.");
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
case "verify": {
|
|
175
|
+
const r = await verifyImport(staging, {
|
|
176
|
+
minRecall: option("min-recall") ? Number(option("min-recall")) : undefined,
|
|
177
|
+
});
|
|
178
|
+
for (const [slug, s] of Object.entries(r.byCollection))
|
|
179
|
+
console.log(` ${slug.padEnd(20)} ${s.found}/${s.planned} documents · ${s.translations} translations`);
|
|
180
|
+
const lines = r.problems.map((p) => `- \`${p.collection}/${p.docId}\` ${p.message}`);
|
|
181
|
+
writeFileSync(
|
|
182
|
+
path.join(importDir(), "verify.md"),
|
|
183
|
+
`# Verify\n\n${r.ok}/${r.checked} documents match the plan.\n\n${lines.join("\n")}\n`,
|
|
184
|
+
);
|
|
185
|
+
console.log(`[kide-wp] ${r.ok}/${r.checked} documents match the plan.`);
|
|
186
|
+
for (const l of lines.slice(0, 30)) console.log(` ${l}`);
|
|
187
|
+
if (r.problems.length) process.exitCode = 1;
|
|
188
|
+
break;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
case "check-live": {
|
|
192
|
+
const minScore = Number(option("min-score") ?? 0.9);
|
|
193
|
+
const started = Date.now();
|
|
194
|
+
const r = await checkLive(staging, {
|
|
195
|
+
baseUrl: option("base"),
|
|
196
|
+
concurrency: option("concurrency") ? Number(option("concurrency")) : undefined,
|
|
197
|
+
limit: option("limit") ? Number(option("limit")) : undefined,
|
|
198
|
+
refresh: flag("refresh"),
|
|
199
|
+
});
|
|
200
|
+
writeFileSync(path.join(importDir(), "live.md"), renderLive(r, minScore));
|
|
201
|
+
const score = liveScore;
|
|
202
|
+
const ok = r.pages.filter((p) => p.status === 200);
|
|
203
|
+
const below = ok.filter((p) => score(p) < minScore);
|
|
204
|
+
const mean = ok.length ? ok.reduce((a, p) => a + score(p), 0) / ok.length : 1;
|
|
205
|
+
console.log(
|
|
206
|
+
`[kide-wp] ${r.pages.length} live pages in ${Date.now() - started}ms · mean ${(mean * 100).toFixed(1)}% · ${below.length} below ${(minScore * 100).toFixed(0)}% · ${ok.filter((p) => p.skipped.length).length} show skipped-source text · ${ok.filter((p) => p.drift.length).length} with text not in the dump · ${r.pages.length - ok.length} not 200 · ${r.failed.length} failed`,
|
|
207
|
+
);
|
|
208
|
+
console.log(`Details: ${out("live.md")}`);
|
|
209
|
+
if (below.length || r.failed.length) process.exitCode = 1;
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
default:
|
|
214
|
+
console.log(USAGE);
|
|
215
|
+
process.exitCode = command ? 1 : 0;
|
|
216
|
+
}
|
|
217
|
+
} catch (e) {
|
|
218
|
+
fail((e as Error).message);
|
|
219
|
+
} finally {
|
|
220
|
+
staging.close();
|
|
221
|
+
}
|
package/src/domain.ts
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The "domain" of a source record: every non-empty leaf path a mapping has to
|
|
3
|
+
* account for (map it or drop it with a reason). Shared by `scout` (profile)
|
|
4
|
+
* and `plan` (coverage ledger) so both speak the same paths:
|
|
5
|
+
*
|
|
6
|
+
* title · meta.hero_title · meta.rows[].label · terms.category
|
|
7
|
+
* block:acf/hero · block:acf/hero.data.heading · shortcode:gallery
|
|
8
|
+
*/
|
|
9
|
+
import { decodeEntities } from "@kidecms/core";
|
|
10
|
+
import { findShortcodes, isAcfBlock, nestAcf, parseBlocks, walkBlocks, type WpBlock } from "./gutenberg";
|
|
11
|
+
import type { StagedRecord } from "./staging";
|
|
12
|
+
|
|
13
|
+
/** Engine-owned fields: used for ids, status, locales — never part of coverage. */
|
|
14
|
+
export const STRUCTURAL = new Set([
|
|
15
|
+
"key",
|
|
16
|
+
"type",
|
|
17
|
+
"id",
|
|
18
|
+
"status",
|
|
19
|
+
"locale",
|
|
20
|
+
"group",
|
|
21
|
+
"modified",
|
|
22
|
+
"permalink",
|
|
23
|
+
"frontPage",
|
|
24
|
+
// url-change records: where the change came from.
|
|
25
|
+
"collection",
|
|
26
|
+
"docId",
|
|
27
|
+
"sourceKey",
|
|
28
|
+
]);
|
|
29
|
+
const VISUAL_ATTRS = new Set([
|
|
30
|
+
"className",
|
|
31
|
+
"align",
|
|
32
|
+
"style",
|
|
33
|
+
"backgroundColor",
|
|
34
|
+
"textColor",
|
|
35
|
+
"gradient",
|
|
36
|
+
"fontSize",
|
|
37
|
+
"fontFamily",
|
|
38
|
+
"layout",
|
|
39
|
+
"lock",
|
|
40
|
+
"metadata",
|
|
41
|
+
"anchor",
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
export const isEmpty = (v: unknown): boolean =>
|
|
45
|
+
v == null ||
|
|
46
|
+
(typeof v === "string" && !v.trim()) ||
|
|
47
|
+
(Array.isArray(v) && v.length === 0) ||
|
|
48
|
+
(typeof v === "object" &&
|
|
49
|
+
!Array.isArray(v) &&
|
|
50
|
+
(Object.keys(v as object).length === 0 ||
|
|
51
|
+
((v as Record<string, unknown>).type === "root" && isEmpty((v as Record<string, unknown>).children))));
|
|
52
|
+
|
|
53
|
+
export const truncate = (v: unknown, n = 80): string => {
|
|
54
|
+
const s = typeof v === "string" ? v : JSON.stringify(v);
|
|
55
|
+
const flat = s.replace(/\s+/g, " ").trim();
|
|
56
|
+
return flat.length > n ? `${flat.slice(0, n - 1)}…` : flat;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
// Objects keyed by data (keywords, ids, dates, URLs) are one value, not hundreds of fields.
|
|
60
|
+
const MAP_LIKE_KEYS = 40;
|
|
61
|
+
const DATA_KEY = /^(https?:\/\/|\/|\d+$|[0-9a-f]{12,}$|\d{4}-\d{2})/i;
|
|
62
|
+
const dataKeyed = (value: object) => Object.keys(value).some((k) => DATA_KEY.test(k));
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* A copy of a WordPress object the editor embedded in block attributes (a media-modal attachment,
|
|
66
|
+
* a REST post): only its id is content; the rest is a stale snapshot of the real record.
|
|
67
|
+
*/
|
|
68
|
+
const snapshotId = (value: Record<string, unknown>): string | undefined => {
|
|
69
|
+
const id = ["ID", "id"].find((k) => typeof value[k] === "number" || /^\d+$/.test(String(value[k] ?? "")));
|
|
70
|
+
const snapshot =
|
|
71
|
+
"nonces" in value || "editLink" in value || "_links" in value || ("mime" in value && "sizes" in value);
|
|
72
|
+
return id && snapshot ? id : undefined;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
const walkLeaves = (value: unknown, path: string, visit: (path: string, value: unknown) => void) => {
|
|
76
|
+
if (isEmpty(value)) return;
|
|
77
|
+
const id =
|
|
78
|
+
value && typeof value === "object" && !Array.isArray(value)
|
|
79
|
+
? snapshotId(value as Record<string, unknown>)
|
|
80
|
+
: undefined;
|
|
81
|
+
if (id) return visit(path ? `${path}.${id}` : id, (value as Record<string, unknown>)[id]);
|
|
82
|
+
if (Array.isArray(value)) {
|
|
83
|
+
for (const item of value) walkLeaves(item, `${path}[]`, visit);
|
|
84
|
+
} else if (
|
|
85
|
+
typeof value === "object" &&
|
|
86
|
+
(!path || (Object.keys(value as object).length <= MAP_LIKE_KEYS && !dataKeyed(value as object)))
|
|
87
|
+
) {
|
|
88
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>))
|
|
89
|
+
walkLeaves(v, path ? `${path}.${k}` : k, visit);
|
|
90
|
+
} else {
|
|
91
|
+
visit(path, value);
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export const blockData = (block: WpBlock): Record<string, unknown> =>
|
|
96
|
+
isAcfBlock(block) ? nestAcf(block.attrs.data as Record<string, unknown>) : {};
|
|
97
|
+
|
|
98
|
+
export const blockAttrs = (block: WpBlock): Record<string, unknown> =>
|
|
99
|
+
Object.fromEntries(
|
|
100
|
+
Object.entries(block.attrs).filter(
|
|
101
|
+
([k]) =>
|
|
102
|
+
!VISUAL_ATTRS.has(k) &&
|
|
103
|
+
!(block.name.startsWith("acf/") && ["data", "name", "mode", "id", "wpClassName"].includes(k)),
|
|
104
|
+
),
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
export const HTML_BLOCKS = new Set(
|
|
108
|
+
[
|
|
109
|
+
"paragraph",
|
|
110
|
+
"heading",
|
|
111
|
+
"list",
|
|
112
|
+
"list-item",
|
|
113
|
+
"quote",
|
|
114
|
+
"pullquote",
|
|
115
|
+
"image",
|
|
116
|
+
"gallery",
|
|
117
|
+
"verse",
|
|
118
|
+
"freeform",
|
|
119
|
+
"buttons",
|
|
120
|
+
"button",
|
|
121
|
+
"file",
|
|
122
|
+
].map((n) => `core/${n}`),
|
|
123
|
+
);
|
|
124
|
+
export const CONTAINER_BLOCKS = new Set(["group", "columns", "column", "row", "stack"].map((n) => `core/${n}`));
|
|
125
|
+
export const EMPTY_BLOCKS = new Set(["spacer", "separator", "more", "nextpage"].map((n) => `core/${n}`));
|
|
126
|
+
/** Core blocks the planner converts itself (see `STANDARD` in plan.ts); their attributes aren't content. */
|
|
127
|
+
export const STANDARD_BLOCKS = new Set(
|
|
128
|
+
[
|
|
129
|
+
"shortcode",
|
|
130
|
+
"cover",
|
|
131
|
+
"details",
|
|
132
|
+
"table",
|
|
133
|
+
"video",
|
|
134
|
+
"embed",
|
|
135
|
+
"audio",
|
|
136
|
+
"html",
|
|
137
|
+
"media-text",
|
|
138
|
+
"code",
|
|
139
|
+
"preformatted",
|
|
140
|
+
].map((n) => `core/${n}`),
|
|
141
|
+
);
|
|
142
|
+
export const CONVERTED = new Set([...HTML_BLOCKS, ...CONTAINER_BLOCKS, ...EMPTY_BLOCKS, ...STANDARD_BLOCKS]);
|
|
143
|
+
|
|
144
|
+
export const isCoreBlock = (name: string) => name.startsWith("core/");
|
|
145
|
+
|
|
146
|
+
/** Blocks of a post, with `core/block` (reusable block) references expanded in place. */
|
|
147
|
+
export const expandBlocks = (content: string, reusable: (ref: number) => string | undefined, depth = 0): WpBlock[] => {
|
|
148
|
+
const expandList = (list: WpBlock[]): { blocks: WpBlock[]; counts: number[] } => {
|
|
149
|
+
const blocks: WpBlock[] = [];
|
|
150
|
+
const counts: number[] = [];
|
|
151
|
+
for (const b of list) {
|
|
152
|
+
const inner =
|
|
153
|
+
b.name === "core/block" && typeof b.attrs.ref === "number" && depth < 5 ? reusable(b.attrs.ref) : undefined;
|
|
154
|
+
const expanded = inner === undefined ? [expandOne(b)] : expandBlocks(inner, reusable, depth + 1);
|
|
155
|
+
blocks.push(...expanded);
|
|
156
|
+
counts.push(expanded.length);
|
|
157
|
+
}
|
|
158
|
+
return { blocks, counts };
|
|
159
|
+
};
|
|
160
|
+
const expandOne = (b: WpBlock): WpBlock => {
|
|
161
|
+
const { blocks, counts } = expandList(b.innerBlocks);
|
|
162
|
+
let i = 0;
|
|
163
|
+
const innerContent = b.innerContent.flatMap((part) => (part === null ? Array(counts[i++]).fill(null) : [part]));
|
|
164
|
+
return { ...b, innerBlocks: blocks, innerContent };
|
|
165
|
+
};
|
|
166
|
+
return expandList(parseBlocks(content)).blocks;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
export const collectDomain = (
|
|
170
|
+
record: StagedRecord,
|
|
171
|
+
reusable: (ref: number) => string | undefined,
|
|
172
|
+
visit: (path: string, value: unknown) => void,
|
|
173
|
+
) => {
|
|
174
|
+
for (const [k, v] of Object.entries(record)) {
|
|
175
|
+
if (STRUCTURAL.has(k)) continue;
|
|
176
|
+
if (k === "terms") {
|
|
177
|
+
for (const [tax, list] of Object.entries((v as Record<string, unknown>) ?? {}))
|
|
178
|
+
if (!isEmpty(list)) visit(`terms.${tax}`, list);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
walkLeaves(v, k, visit);
|
|
182
|
+
}
|
|
183
|
+
if (typeof record.content === "string" && record.content.trim()) {
|
|
184
|
+
for (const block of walkBlocks(expandBlocks(record.content, reusable))) {
|
|
185
|
+
if (block.name === "core/block") continue;
|
|
186
|
+
const base = `block:${block.name}`;
|
|
187
|
+
visit(base, block.name);
|
|
188
|
+
// Converted core blocks carry layout attributes only; any other core block's attributes (rss feed
|
|
189
|
+
// URL, query loop settings) are content a mapping has to account for.
|
|
190
|
+
if (CONVERTED.has(block.name)) continue;
|
|
191
|
+
walkLeaves(blockData(block), `${base}.data`, visit);
|
|
192
|
+
walkLeaves(blockAttrs(block), `${base}.attrs`, visit);
|
|
193
|
+
if (!isAcfBlock(block) && htmlText(block.innerHTML)) visit(`${base}.innerHTML`, block.innerHTML);
|
|
194
|
+
}
|
|
195
|
+
// Brackets inside comments (`<!--[if lte IE 8]>`, block attribute JSON) aren't shortcodes.
|
|
196
|
+
const text = stripComments(record.content);
|
|
197
|
+
for (const name of new Set(findShortcodes(text))) visit(`shortcode:${name}`, name);
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
/** Text that can't hold a shortcode: comments (block attribute JSON) and script/style bodies (`i[e]`). */
|
|
202
|
+
export const stripComments = (html: string) =>
|
|
203
|
+
html.replace(/<!--[\s\S]*?-->/g, "").replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, "");
|
|
204
|
+
|
|
205
|
+
// --- glob matching for `drop` --------------------------------------------------
|
|
206
|
+
|
|
207
|
+
const globCache = new Map<string, RegExp>();
|
|
208
|
+
export const matchesPath = (pattern: string, path: string): boolean => {
|
|
209
|
+
if (pattern === path || path.startsWith(`${pattern}.`) || path.startsWith(`${pattern}[]`)) return true;
|
|
210
|
+
if (!pattern.includes("*")) return false;
|
|
211
|
+
let re = globCache.get(pattern);
|
|
212
|
+
if (!re) {
|
|
213
|
+
re = new RegExp(`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}(\\.|\\[\\]|$)`);
|
|
214
|
+
globCache.set(pattern, re);
|
|
215
|
+
}
|
|
216
|
+
return re.test(path);
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
export const isCovered = (path: string, reads: Set<string>): boolean => {
|
|
220
|
+
if (reads.has(path)) return true;
|
|
221
|
+
for (const r of reads) if (r.startsWith(`${path}.`) || r.startsWith(`${path}[]`)) return true;
|
|
222
|
+
return false;
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
// --- text helpers ---------------------------------------------------------------
|
|
226
|
+
|
|
227
|
+
export const htmlText = (html: string): string =>
|
|
228
|
+
decodeEntities(
|
|
229
|
+
html
|
|
230
|
+
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
231
|
+
.replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ")
|
|
232
|
+
.replace(/<[^>]+>/g, " "),
|
|
233
|
+
)
|
|
234
|
+
.replace(/\s+/g, " ")
|
|
235
|
+
.trim();
|
|
236
|
+
|
|
237
|
+
export const words = (text: string): string[] => text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
238
|
+
|
|
239
|
+
/** Every string leaf in a value (rich-text text nodes, block fields, plain fields). */
|
|
240
|
+
export const outputStrings = (value: unknown, out: string[] = []): string[] => {
|
|
241
|
+
if (typeof value === "string") out.push(value);
|
|
242
|
+
else if (Array.isArray(value)) for (const v of value) outputStrings(v, out);
|
|
243
|
+
else if (value && typeof value === "object") for (const v of Object.values(value)) outputStrings(v, out);
|
|
244
|
+
return out;
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
/** Share of expected words (multiset) that appear in the output. 1 when nothing was expected. */
|
|
248
|
+
export const textRecall = (
|
|
249
|
+
expected: string[],
|
|
250
|
+
output: string[],
|
|
251
|
+
mode: "multiset" | "set" = "multiset",
|
|
252
|
+
): { recall: number; missing: string[] } => {
|
|
253
|
+
const have = new Map<string, number>();
|
|
254
|
+
for (const w of output.flatMap((s) => words(s))) have.set(w, mode === "set" ? Infinity : (have.get(w) ?? 0) + 1);
|
|
255
|
+
const all = expected.flatMap((s) => words(s));
|
|
256
|
+
const exp = mode === "set" ? [...new Set(all)] : all;
|
|
257
|
+
if (!exp.length) return { recall: 1, missing: [] };
|
|
258
|
+
let hit = 0;
|
|
259
|
+
const missing: string[] = [];
|
|
260
|
+
for (const w of exp) {
|
|
261
|
+
const n = have.get(w) ?? 0;
|
|
262
|
+
if (n > 0) {
|
|
263
|
+
hit++;
|
|
264
|
+
have.set(w, n - 1);
|
|
265
|
+
} else if (missing.length < 12) missing.push(w);
|
|
266
|
+
}
|
|
267
|
+
return { recall: hit / exp.length, missing };
|
|
268
|
+
};
|
package/src/gutenberg.ts
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/** Gutenberg blocks, shortcodes and autop via WordPress's own packages, plus ACF flat-meta reconstruction. */
|
|
2
|
+
import { autop as wpautop } from "@wordpress/autop";
|
|
3
|
+
import { parse } from "@wordpress/block-serialization-default-parser";
|
|
4
|
+
import { replace as replaceShortcode } from "@wordpress/shortcode";
|
|
5
|
+
|
|
6
|
+
export type WpBlock = {
|
|
7
|
+
name: string;
|
|
8
|
+
attrs: Record<string, unknown>;
|
|
9
|
+
innerHTML: string;
|
|
10
|
+
innerBlocks: WpBlock[];
|
|
11
|
+
/** innerHTML segments with `null` where each inner block sits. */
|
|
12
|
+
innerContent: Array<string | null>;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/** Parse post_content into blocks (WordPress's own parser). HTML outside any block becomes `core/freeform`. */
|
|
16
|
+
type ParsedBlock = ReturnType<typeof parse>[number];
|
|
17
|
+
|
|
18
|
+
export const parseBlocks = (content: string): WpBlock[] => {
|
|
19
|
+
const convert = (b: ParsedBlock): WpBlock => ({
|
|
20
|
+
name: b.blockName ?? "core/freeform",
|
|
21
|
+
attrs: (b.attrs ?? {}) as Record<string, unknown>,
|
|
22
|
+
innerHTML: b.innerHTML,
|
|
23
|
+
innerBlocks: b.innerBlocks.map(convert),
|
|
24
|
+
innerContent: b.innerContent,
|
|
25
|
+
});
|
|
26
|
+
return parse(content)
|
|
27
|
+
.filter((b) => b.blockName || b.innerHTML.trim())
|
|
28
|
+
.map(convert);
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** A block's full HTML with inner blocks rendered in place. */
|
|
32
|
+
export const blockHtml = (block: WpBlock): string => {
|
|
33
|
+
let i = 0;
|
|
34
|
+
return block.innerContent.map((part) => part ?? blockHtml(block.innerBlocks[i++])).join("");
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export const isAcfBlock = (block: WpBlock) => block.name.startsWith("acf/") && typeof block.attrs.data === "object";
|
|
38
|
+
|
|
39
|
+
export const walkBlocks = function* (blocks: WpBlock[]): Generator<WpBlock> {
|
|
40
|
+
for (const block of blocks) {
|
|
41
|
+
yield block;
|
|
42
|
+
yield* walkBlocks(block.innerBlocks);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const SHORTCODE = /\[(\/)?([a-z][a-z0-9_-]*)((?:\s[^\]]*)?)(\/)?\]/gi;
|
|
47
|
+
|
|
48
|
+
export const findShortcodes = (html: string): string[] =>
|
|
49
|
+
[...html.matchAll(SHORTCODE)].filter((m) => !m[1]).map((m) => m[2].toLowerCase());
|
|
50
|
+
|
|
51
|
+
/** HTML, or content nodes (spliced in where the shortcode stood). */
|
|
52
|
+
export type ShortcodeHandler = (attrs: Record<string, string>, inner: string) => string | object | object[];
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Replace shortcodes whose name has a handler; others are left in place. A handler returning
|
|
56
|
+
* content nodes leaves the placeholder `embed` gives back, for the caller to splice them in.
|
|
57
|
+
*/
|
|
58
|
+
export const replaceShortcodes = (
|
|
59
|
+
html: string,
|
|
60
|
+
handlers: Record<string, ShortcodeHandler>,
|
|
61
|
+
onHandled: (name: string) => void,
|
|
62
|
+
embed: (nodes: object[]) => string = () => "",
|
|
63
|
+
): string => {
|
|
64
|
+
let out = html;
|
|
65
|
+
for (const name of Object.keys(handlers))
|
|
66
|
+
out = replaceShortcode(name, out, (shortcode) => {
|
|
67
|
+
onHandled(name);
|
|
68
|
+
const { named, numeric } = shortcode.attrs;
|
|
69
|
+
const attrs = { ...Object.fromEntries(numeric.map((v, i) => [String(i), v])), ...named } as Record<
|
|
70
|
+
string,
|
|
71
|
+
string
|
|
72
|
+
>;
|
|
73
|
+
const result = handlers[name](attrs, shortcode.content ?? "");
|
|
74
|
+
return typeof result === "string" ? result : embed(Array.isArray(result) ? result : [result]);
|
|
75
|
+
});
|
|
76
|
+
return out;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/** WordPress's `wpautop`: classic-editor content separates paragraphs with blank lines, not tags. */
|
|
80
|
+
export const autop = (html: string): string => (html.trim() ? wpautop(html) : html);
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Rebuild ACF's flattened storage into nested values: a repeater is a row count
|
|
84
|
+
* at `name` plus `name_<i>_<sub>` keys; flexible content stores the layout names
|
|
85
|
+
* as an array instead of a count. `_name` companions (`field_…` keys) are dropped.
|
|
86
|
+
*/
|
|
87
|
+
export const nestAcf = (flat: Record<string, unknown>, listFields?: Set<string>): Record<string, unknown> => {
|
|
88
|
+
const keys = Object.keys(flat)
|
|
89
|
+
.filter((k) => !(k.startsWith("_") && typeof flat[k] === "string" && /^field_/.test(flat[k] as string)))
|
|
90
|
+
.sort((a, b) => a.length - b.length);
|
|
91
|
+
const consumed = new Set<string>();
|
|
92
|
+
const out: Record<string, unknown> = {};
|
|
93
|
+
for (const key of keys) {
|
|
94
|
+
if (consumed.has(key)) continue;
|
|
95
|
+
const value = flat[key];
|
|
96
|
+
if (value === "" && listFields?.has(String(flat[`_${key}`]))) {
|
|
97
|
+
out[key] = [];
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const layouts = Array.isArray(value) && value.every((v) => typeof v === "string") ? (value as string[]) : null;
|
|
101
|
+
const count =
|
|
102
|
+
typeof value === "number" || (typeof value === "string" && /^\d+$/.test(value)) ? Number(value) : null;
|
|
103
|
+
const hasRows = keys.some((k) => k.startsWith(`${key}_0_`));
|
|
104
|
+
if ((layouts || count !== null) && hasRows) {
|
|
105
|
+
const n = layouts ? layouts.length : count!;
|
|
106
|
+
const rows: Record<string, unknown>[] = [];
|
|
107
|
+
for (let i = 0; i < n; i++) {
|
|
108
|
+
const prefix = `${key}_${i}_`;
|
|
109
|
+
const sub: Record<string, unknown> = {};
|
|
110
|
+
for (const k of keys) {
|
|
111
|
+
if (!k.startsWith(prefix)) continue;
|
|
112
|
+
sub[k.slice(prefix.length)] = flat[k];
|
|
113
|
+
consumed.add(k);
|
|
114
|
+
}
|
|
115
|
+
const row = nestAcf(sub, listFields);
|
|
116
|
+
if (layouts) row.acf_fc_layout = layouts[i];
|
|
117
|
+
rows.push(row);
|
|
118
|
+
}
|
|
119
|
+
out[key] = rows;
|
|
120
|
+
} else {
|
|
121
|
+
out[key] = value;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return out;
|
|
125
|
+
};
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { defineMapping, skipSource } from "./mapping";
|
|
2
|
+
export type { BlockHandler, BlockInput, MapContext, Mapping, MenuItem, SourceRecord, TermNode } from "./mapping";
|
|
3
|
+
export { termSlugs } from "./structures";
|
|
4
|
+
export { parseBlocks, nestAcf } from "./gutenberg";
|
|
5
|
+
export { readMysqlDump } from "./mysql-dump";
|
|
6
|
+
export { maybeUnserialize } from "./php";
|
|
7
|
+
export { termParts } from "./terms";
|