@kidecms/wordpress 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/bin/kide-wp.mjs +6 -0
- package/package.json +37 -3
- package/src/apply.ts +332 -0
- package/src/automap.ts +143 -0
- package/src/cli.ts +221 -0
- package/src/domain.ts +268 -0
- package/src/gutenberg.ts +125 -0
- package/src/index.ts +7 -0
- package/src/live.ts +282 -0
- package/src/mapping.ts +170 -0
- package/src/mysql-dump.ts +202 -0
- package/src/php.ts +62 -0
- package/src/plan.ts +1467 -0
- package/src/profile.ts +123 -0
- package/src/report.ts +128 -0
- package/src/staging.ts +77 -0
- package/src/structures.ts +199 -0
- package/src/terms.ts +6 -0
- package/src/verify.ts +156 -0
- package/src/wordpress.ts +472 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matti Hernesniemi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @kidecms/wordpress
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
WordPress → Kide CMS importer. Scout a MySQL dump, describe the target with `defineMapping`, then
|
|
4
|
+
`kide-wp plan → apply → verify → check-live`. The playbook is `MIGRATING.md` in a Kide project.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pnpm add -D @kidecms/wordpress
|
|
8
|
+
pnpm exec kide-wp scout wordpress.sql.gz
|
|
9
|
+
```
|
package/bin/kide-wp.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,40 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kidecms/wordpress",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "WordPress → Kide CMS importer: scout a MySQL dump, map it with defineMapping, then kide-wp plan → apply → verify → check-live.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/mhernesniemi/kide-cms.git",
|
|
10
|
+
"directory": "packages/wordpress"
|
|
11
|
+
},
|
|
12
|
+
"bin": {
|
|
13
|
+
"kide-wp": "./bin/kide-wp.mjs"
|
|
14
|
+
},
|
|
15
|
+
"exports": {
|
|
16
|
+
".": "./src/index.ts"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"bin",
|
|
20
|
+
"src",
|
|
21
|
+
"!src/__tests__"
|
|
22
|
+
],
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"@wordpress/autop": "5.1.0",
|
|
25
|
+
"@wordpress/block-serialization-default-parser": "6.1.0",
|
|
26
|
+
"@wordpress/shortcode": "5.1.0",
|
|
27
|
+
"tsx": "^4.23.15"
|
|
28
|
+
},
|
|
29
|
+
"peerDependencies": {
|
|
30
|
+
"@kidecms/core": ">=0.41.0",
|
|
31
|
+
"better-sqlite3": "^12.0.0 || ^13.0.0"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@kidecms/core": "0.42.0"
|
|
35
|
+
},
|
|
36
|
+
"scripts": {
|
|
37
|
+
"check": "tsc --noEmit -p .",
|
|
38
|
+
"test": "vitest run"
|
|
39
|
+
}
|
|
6
40
|
}
|
package/src/apply.ts
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `kide-wp apply`: write the stored plan incrementally. Unchanged documents
|
|
3
|
+
* (same hash, still present) are skipped, media is uploaded once in parallel and
|
|
4
|
+
* remembered, writes are batched in transactions with a savepoint per document,
|
|
5
|
+
* and documents the importer created earlier but the plan no longer contains are
|
|
6
|
+
* removed. Search is rebuilt once at the end.
|
|
7
|
+
*/
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
9
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
|
|
13
|
+
import { getSchema, getTranslatableFieldNames } from "@kidecms/core";
|
|
14
|
+
import { createCmsContext } from "@kidecms/core/context";
|
|
15
|
+
import { MEDIA_TOKEN, tokenKey } from "./plan";
|
|
16
|
+
import { loadPlan } from "./report";
|
|
17
|
+
import { getMeta, setMeta, type Staging } from "./staging";
|
|
18
|
+
|
|
19
|
+
export type ApplyReport = {
|
|
20
|
+
created: number;
|
|
21
|
+
updated: number;
|
|
22
|
+
unchanged: number;
|
|
23
|
+
deleted: number;
|
|
24
|
+
failed: Array<{ docId: string; message: string }>;
|
|
25
|
+
/** `missing`: the source itself answers 4xx (the file was deleted); references to it are dropped. */
|
|
26
|
+
media: {
|
|
27
|
+
uploaded: number;
|
|
28
|
+
reused: number;
|
|
29
|
+
missing: Array<{ key: string; message: string }>;
|
|
30
|
+
failed: Array<{ key: string; message: string }>;
|
|
31
|
+
};
|
|
32
|
+
durationMs: number;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const MIME: Record<string, string> = {
|
|
36
|
+
jpg: "image/jpeg",
|
|
37
|
+
jpeg: "image/jpeg",
|
|
38
|
+
png: "image/png",
|
|
39
|
+
gif: "image/gif",
|
|
40
|
+
webp: "image/webp",
|
|
41
|
+
avif: "image/avif",
|
|
42
|
+
svg: "image/svg+xml",
|
|
43
|
+
pdf: "application/pdf",
|
|
44
|
+
mp4: "video/mp4",
|
|
45
|
+
webm: "video/webm",
|
|
46
|
+
mp3: "audio/mpeg",
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const pool = async <T>(items: T[], size: number, run: (item: T) => Promise<void>) => {
|
|
50
|
+
let i = 0;
|
|
51
|
+
await Promise.all(
|
|
52
|
+
Array.from({ length: Math.min(size, items.length) }, async () => {
|
|
53
|
+
while (i < items.length) await run(items[i++]);
|
|
54
|
+
}),
|
|
55
|
+
);
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Live sites drop the odd connection under parallel load; a 4xx is final, anything else is retried.
|
|
59
|
+
const download = async (url: string, attempts = 4): Promise<Uint8Array> => {
|
|
60
|
+
for (let i = 1; ; i++) {
|
|
61
|
+
try {
|
|
62
|
+
// Generous: a large video over a slow live site is still one file.
|
|
63
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(300_000) });
|
|
64
|
+
if (res.status >= 400 && res.status < 500)
|
|
65
|
+
throw Object.assign(new Error(`HTTP ${res.status} for ${url}`), { final: true });
|
|
66
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
|
67
|
+
return new Uint8Array(await res.arrayBuffer());
|
|
68
|
+
} catch (e) {
|
|
69
|
+
if ((e as { final?: boolean }).final || i >= attempts) throw e;
|
|
70
|
+
await new Promise((r) => setTimeout(r, 500 * 2 ** i));
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/** Downloads are kept across projects and fresh runs, so the source site is fetched once per file. */
|
|
76
|
+
const mediaCache = () =>
|
|
77
|
+
path.resolve(process.env.KIDE_MEDIA_CACHE ?? path.join(homedir(), ".cache", "kide", "import-media"));
|
|
78
|
+
|
|
79
|
+
// A 4xx is cached too (`.gone`), so a fresh project doesn't ask the source again; `apply --all` does.
|
|
80
|
+
const fetchCached = async (url: string, retryGone = false): Promise<Uint8Array> => {
|
|
81
|
+
const file = path.join(mediaCache(), createHash("sha1").update(url).digest("hex"));
|
|
82
|
+
const cached = await readFile(file).catch(() => null);
|
|
83
|
+
if (cached) return new Uint8Array(cached);
|
|
84
|
+
const gone = await readFile(`${file}.gone`, "utf8").catch(() => null);
|
|
85
|
+
if (gone !== null && !retryGone) throw Object.assign(new Error(gone), { final: true });
|
|
86
|
+
await mkdir(path.dirname(file), { recursive: true });
|
|
87
|
+
try {
|
|
88
|
+
const bytes = await download(url);
|
|
89
|
+
await writeFile(file, bytes);
|
|
90
|
+
return bytes;
|
|
91
|
+
} catch (e) {
|
|
92
|
+
if ((e as { final?: boolean }).final) await writeFile(`${file}.gone`, (e as Error).message);
|
|
93
|
+
throw e;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const SYSTEM = { _system: true, _skipSearch: true };
|
|
98
|
+
const snakeCase = (value: string) =>
|
|
99
|
+
value
|
|
100
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
101
|
+
.replace(/[^a-zA-Z0-9]+/g, "_")
|
|
102
|
+
.toLowerCase();
|
|
103
|
+
|
|
104
|
+
export const applyImport = async (
|
|
105
|
+
staging: Staging,
|
|
106
|
+
options: { force?: boolean; all?: boolean; prune?: boolean; concurrency?: number; batchSize?: number } = {},
|
|
107
|
+
): Promise<ApplyReport> => {
|
|
108
|
+
const started = Date.now();
|
|
109
|
+
const plan = loadPlan(staging);
|
|
110
|
+
if (!plan) throw new Error("No plan yet — run `kide-wp plan` first.");
|
|
111
|
+
if (plan.blocking.length && !options.force)
|
|
112
|
+
throw new Error(
|
|
113
|
+
`The plan is blocked (${plan.blocking.length} problem(s), see .kide/import/ledger.md). Fix them or pass --force.`,
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
const { cms, db, assets, config, reindex, dispose } = await createCmsContext();
|
|
117
|
+
const drafts = new Set(config.collections.filter((c) => c.drafts).map((c) => c.slug));
|
|
118
|
+
const api = cms as unknown as Record<string, any>;
|
|
119
|
+
const sqlite = (db as any).$client as import("better-sqlite3").Database;
|
|
120
|
+
const report: ApplyReport = {
|
|
121
|
+
created: 0,
|
|
122
|
+
updated: 0,
|
|
123
|
+
unchanged: 0,
|
|
124
|
+
deleted: 0,
|
|
125
|
+
failed: [],
|
|
126
|
+
media: { uploaded: 0, reused: 0, missing: [], failed: [] },
|
|
127
|
+
durationMs: 0,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
// A field added or made translatable without `cms:push` has no column, and writes drop it silently.
|
|
132
|
+
// Compare with the config itself: the generated schema is just as stale until the push.
|
|
133
|
+
const columns = (table: string) =>
|
|
134
|
+
new Set((sqlite.prepare(`PRAGMA table_info("${table}")`).all() as Array<{ name: string }>).map((c) => c.name));
|
|
135
|
+
const behind: string[] = [];
|
|
136
|
+
for (const slug of new Set(plan.docs.map((d) => d.collection))) {
|
|
137
|
+
const collection = config.collections.find((c) => c.slug === slug);
|
|
138
|
+
if (!collection) continue;
|
|
139
|
+
const main = `cms_${snakeCase(slug)}`;
|
|
140
|
+
const translatable = config.locales ? getTranslatableFieldNames(collection) : [];
|
|
141
|
+
for (const [table, fields] of [
|
|
142
|
+
[main, Object.keys(collection.fields)],
|
|
143
|
+
[`${main}_translations`, translatable],
|
|
144
|
+
] as const) {
|
|
145
|
+
if (!fields.length) continue;
|
|
146
|
+
const have = columns(table);
|
|
147
|
+
for (const field of fields) if (!have.has(snakeCase(field))) behind.push(`${table}.${snakeCase(field)}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (behind.length)
|
|
151
|
+
throw new Error(
|
|
152
|
+
`The database schema is behind the config (missing ${behind.slice(0, 5).join(", ")}). Run \`pnpm cms:push\`, then apply again.`,
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
const existing = new Map<string, Set<string>>();
|
|
156
|
+
for (const slug of new Set([...plan.docs.map((d) => d.collection)])) {
|
|
157
|
+
const main = getSchema().cmsTables[slug]?.main;
|
|
158
|
+
const rows = main ? ((await db.select({ _id: main._id }).from(main)) as Array<{ _id: string }>) : [];
|
|
159
|
+
existing.set(slug, new Set(rows.map((r) => r._id)));
|
|
160
|
+
}
|
|
161
|
+
const applied = new Map(
|
|
162
|
+
(
|
|
163
|
+
staging.prepare("SELECT doc_id, collection, hash FROM applied").all() as Array<{
|
|
164
|
+
doc_id: string;
|
|
165
|
+
collection: string;
|
|
166
|
+
hash: string;
|
|
167
|
+
}>
|
|
168
|
+
).map((r) => [r.doc_id, r]),
|
|
169
|
+
);
|
|
170
|
+
const pending = plan.docs.filter(
|
|
171
|
+
(d) => options.all || applied.get(d.docId)?.hash !== d.hash || !existing.get(d.collection)?.has(d.docId),
|
|
172
|
+
);
|
|
173
|
+
report.unchanged = plan.docs.length - pending.length;
|
|
174
|
+
|
|
175
|
+
// --- media -------------------------------------------------------------
|
|
176
|
+
const resolved = new Map(
|
|
177
|
+
(
|
|
178
|
+
staging.prepare("SELECT key, storage_path FROM media_state").all() as Array<{
|
|
179
|
+
key: string;
|
|
180
|
+
storage_path: string;
|
|
181
|
+
}>
|
|
182
|
+
).map((r) => [r.key, r.storage_path]),
|
|
183
|
+
);
|
|
184
|
+
const assetRows = (await db.select({ p: getSchema().cmsAssets.storagePath }).from(getSchema().cmsAssets)) as Array<{
|
|
185
|
+
p: string;
|
|
186
|
+
}>;
|
|
187
|
+
const livePaths = new Set(assetRows.map((r) => r.p));
|
|
188
|
+
for (const [key, p] of resolved) if (!livePaths.has(p)) resolved.delete(key);
|
|
189
|
+
|
|
190
|
+
const needed = [...new Set(pending.flatMap((d) => d.media))];
|
|
191
|
+
// Files the source answered 4xx for stay missing until `--all` asks again.
|
|
192
|
+
const knownMissing = new Set(options.all ? [] : (JSON.parse(getMeta(staging, "missingMedia") ?? "[]") as string[]));
|
|
193
|
+
const toUpload = needed.filter((k) => !resolved.has(k) && !knownMissing.has(k));
|
|
194
|
+
report.media.reused = needed.length - toUpload.length;
|
|
195
|
+
const uploadsDir = getMeta(staging, "uploadsDir");
|
|
196
|
+
const remember = staging.prepare(
|
|
197
|
+
"INSERT OR REPLACE INTO media_state (key, storage_path, applied_at) VALUES (?, ?, ?)",
|
|
198
|
+
);
|
|
199
|
+
await pool(toUpload, options.concurrency ?? 12, async (key) => {
|
|
200
|
+
const m = plan.media[key];
|
|
201
|
+
try {
|
|
202
|
+
if (!m) throw new Error("not in plan");
|
|
203
|
+
let bytes: Uint8Array;
|
|
204
|
+
if (uploadsDir && m.file) bytes = await readFile(path.join(uploadsDir, m.file));
|
|
205
|
+
else bytes = await fetchCached(m.url, options.all);
|
|
206
|
+
const name = decodeURIComponent(path.basename(m.file ?? new URL(m.url).pathname));
|
|
207
|
+
const ext = name.split(".").pop()?.toLowerCase() ?? "";
|
|
208
|
+
const file = new File([bytes as BlobPart], name, { type: m.mime ?? MIME[ext] ?? "application/octet-stream" });
|
|
209
|
+
const asset = await assets.upload(file, { alt: m.alt ?? undefined, dedupe: true });
|
|
210
|
+
resolved.set(key, asset.storagePath);
|
|
211
|
+
remember.run(key, asset.storagePath, new Date().toISOString());
|
|
212
|
+
report.media.uploaded++;
|
|
213
|
+
} catch (e) {
|
|
214
|
+
const gone = (e as { final?: boolean }).final || (e as { code?: string }).code === "ENOENT";
|
|
215
|
+
report.media[gone ? "missing" : "failed"].push({ key, message: (e as Error).message });
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
const gone = new Set<string>([
|
|
219
|
+
...[...knownMissing].filter((k) => !resolved.has(k)),
|
|
220
|
+
...report.media.missing.map((m) => m.key),
|
|
221
|
+
]);
|
|
222
|
+
setMeta(staging, "missingMedia", JSON.stringify([...gone]));
|
|
223
|
+
|
|
224
|
+
// --- documents ----------------------------------------------------------
|
|
225
|
+
const markApplied = staging.prepare(
|
|
226
|
+
"INSERT OR REPLACE INTO applied (doc_id, collection, hash, applied_at) VALUES (?, ?, ?, ?)",
|
|
227
|
+
);
|
|
228
|
+
const isGone = (v: unknown) => {
|
|
229
|
+
const m = typeof v === "string" ? /^\/uploads\/\.kide-media\/([A-Za-z0-9_-]+)$/.exec(v) : null;
|
|
230
|
+
return !!m && gone.has(tokenKey(m[1]));
|
|
231
|
+
};
|
|
232
|
+
// Images go, links keep their text, media fields and link values become null.
|
|
233
|
+
const dropGone = (value: unknown): unknown => {
|
|
234
|
+
if (Array.isArray(value))
|
|
235
|
+
return value.filter((v) => !isGone(v) && !(v?.type === "image" && isGone(v.src))).map(dropGone);
|
|
236
|
+
if (!value || typeof value !== "object") return isGone(value) ? null : value;
|
|
237
|
+
if (isGone((value as Record<string, unknown>).url)) return null;
|
|
238
|
+
return Object.fromEntries(
|
|
239
|
+
Object.entries(value)
|
|
240
|
+
.filter(([k, v]) => !(k === "href" && isGone(v)))
|
|
241
|
+
.map(([k, v]) => [k, dropGone(v)]),
|
|
242
|
+
);
|
|
243
|
+
};
|
|
244
|
+
const resolveTokens = (value: unknown) => {
|
|
245
|
+
const missing: string[] = [];
|
|
246
|
+
const json = JSON.stringify(gone.size ? dropGone(value) : value).replace(MEDIA_TOKEN, (_, encoded: string) => {
|
|
247
|
+
const p = resolved.get(tokenKey(encoded));
|
|
248
|
+
if (!p) missing.push(tokenKey(encoded));
|
|
249
|
+
return p ?? "";
|
|
250
|
+
});
|
|
251
|
+
return { value: JSON.parse(json), missing };
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
const batchSize = options.batchSize ?? 100;
|
|
255
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
256
|
+
const batch = pending.slice(i, i + batchSize);
|
|
257
|
+
const done: typeof batch = [];
|
|
258
|
+
sqlite.exec("BEGIN");
|
|
259
|
+
try {
|
|
260
|
+
for (const doc of batch) {
|
|
261
|
+
const data = resolveTokens(doc.data);
|
|
262
|
+
const translations = resolveTokens(doc.translations);
|
|
263
|
+
const missing = [...data.missing, ...translations.missing];
|
|
264
|
+
if (missing.length) {
|
|
265
|
+
report.failed.push({ docId: doc.docId, message: `media not uploaded: ${missing.slice(0, 3).join(", ")}` });
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
sqlite.exec("SAVEPOINT doc");
|
|
269
|
+
try {
|
|
270
|
+
const isNew = !existing.get(doc.collection)?.has(doc.docId);
|
|
271
|
+
await api[doc.collection].upsert(data.value, SYSTEM);
|
|
272
|
+
for (const [locale, overlay] of Object.entries(
|
|
273
|
+
translations.value as Record<string, Record<string, unknown>>,
|
|
274
|
+
))
|
|
275
|
+
await api[doc.collection].upsertTranslation(
|
|
276
|
+
doc.docId,
|
|
277
|
+
locale,
|
|
278
|
+
{
|
|
279
|
+
// Fields the plan left out inherit the original, so clear any earlier copy.
|
|
280
|
+
...Object.fromEntries(
|
|
281
|
+
getTranslatableFieldNames(api.meta.getCollection(doc.collection)).map((f) => [f, null]),
|
|
282
|
+
),
|
|
283
|
+
...overlay,
|
|
284
|
+
...(doc.data._status === "published" ? { _status: "published" } : {}),
|
|
285
|
+
},
|
|
286
|
+
SYSTEM,
|
|
287
|
+
);
|
|
288
|
+
// Updating a published document parks the old version as its live snapshot; republish so the site shows the import.
|
|
289
|
+
if (!isNew && doc.data._status === "published" && drafts.has(doc.collection))
|
|
290
|
+
await api[doc.collection].publish(doc.docId, SYSTEM);
|
|
291
|
+
sqlite.exec("RELEASE doc");
|
|
292
|
+
if (isNew) report.created++;
|
|
293
|
+
else report.updated++;
|
|
294
|
+
done.push(doc);
|
|
295
|
+
} catch (e) {
|
|
296
|
+
sqlite.exec("ROLLBACK TO doc");
|
|
297
|
+
sqlite.exec("RELEASE doc");
|
|
298
|
+
report.failed.push({ docId: doc.docId, message: (e as Error).message });
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
sqlite.exec("COMMIT");
|
|
302
|
+
} catch (e) {
|
|
303
|
+
if (sqlite.inTransaction) sqlite.exec("ROLLBACK");
|
|
304
|
+
throw e;
|
|
305
|
+
}
|
|
306
|
+
const now = new Date().toISOString();
|
|
307
|
+
for (const doc of done) markApplied.run(doc.docId, doc.collection, doc.hash, now);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// --- prune --------------------------------------------------------------
|
|
311
|
+
if (options.prune ?? true) {
|
|
312
|
+
const planned = new Set(plan.docs.map((d) => d.docId));
|
|
313
|
+
const forget = staging.prepare("DELETE FROM applied WHERE doc_id = ?");
|
|
314
|
+
for (const [docId, row] of applied) {
|
|
315
|
+
if (planned.has(docId)) continue;
|
|
316
|
+
try {
|
|
317
|
+
if (api[row.collection]) await api[row.collection].delete(docId, SYSTEM);
|
|
318
|
+
forget.run(docId);
|
|
319
|
+
report.deleted++;
|
|
320
|
+
} catch (e) {
|
|
321
|
+
report.failed.push({ docId, message: `delete failed: ${(e as Error).message}` });
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (report.created || report.updated || report.deleted) await reindex();
|
|
327
|
+
} finally {
|
|
328
|
+
await dispose();
|
|
329
|
+
}
|
|
330
|
+
report.durationMs = Date.now() - started;
|
|
331
|
+
return report;
|
|
332
|
+
};
|
package/src/automap.ts
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mapping by shape: the target's declared fields say what to read and how to convert it, so a
|
|
3
|
+
* mapping only spells out what differs. Reads go through the tracked record, so whatever isn't
|
|
4
|
+
* consumed stays unaccounted in the ledger.
|
|
5
|
+
*/
|
|
6
|
+
import type { CollectionConfig, FieldConfig } from "@kidecms/core";
|
|
7
|
+
import type { MapContext, SourceRecord } from "./mapping";
|
|
8
|
+
|
|
9
|
+
const snake = (name: string) => name.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
|
|
10
|
+
const empty = (v: unknown) => v === undefined || v === null || v === "" || (Array.isArray(v) && !v.length);
|
|
11
|
+
// Reading a tracked key only marks it when filled, so trying both spellings is free.
|
|
12
|
+
// `image` also reads `imageId` / `imageUrl` / `image_id`, as block attributes often name them.
|
|
13
|
+
const pick = (source: Record<string, any> | undefined, name: string) =>
|
|
14
|
+
source == null
|
|
15
|
+
? undefined
|
|
16
|
+
: (source[name] ??
|
|
17
|
+
source[snake(name)] ??
|
|
18
|
+
source[`${name}Id`] ??
|
|
19
|
+
source[`${snake(name)}_id`] ??
|
|
20
|
+
source[`${name}Url`]);
|
|
21
|
+
|
|
22
|
+
/** One ACF/WordPress value → the declared field's value shape. `undefined` leaves the field out. */
|
|
23
|
+
export const convertField = (field: FieldConfig, value: any, ctx: MapContext, name = ""): unknown => {
|
|
24
|
+
if (empty(value)) return undefined;
|
|
25
|
+
const component = field.admin?.component;
|
|
26
|
+
switch (field.type) {
|
|
27
|
+
case "image":
|
|
28
|
+
return ctx.media(value);
|
|
29
|
+
case "richText":
|
|
30
|
+
return ctx.richText(value);
|
|
31
|
+
case "content":
|
|
32
|
+
return ctx.content(value);
|
|
33
|
+
case "boolean":
|
|
34
|
+
return value === true || value === 1 || value === "1" || value === "true";
|
|
35
|
+
case "number":
|
|
36
|
+
return Number(value);
|
|
37
|
+
case "date":
|
|
38
|
+
return ctx.date(value) ?? undefined;
|
|
39
|
+
case "relation": {
|
|
40
|
+
// A reference to something not imported (deleted, draft) is left out, not a blocking issue.
|
|
41
|
+
const ids = (Array.isArray(value) ? value : [value]).filter((v) => !empty(v));
|
|
42
|
+
const refs = ids.map((id) => ctx.ref(id, { optional: true }));
|
|
43
|
+
const lost = ids.filter((_, i) => !refs[i]);
|
|
44
|
+
if (lost.length) ctx.warn(`${name}: ${lost.length} reference(s) to records that aren't imported were left out`);
|
|
45
|
+
const kept = refs.filter((r): r is string => !!r);
|
|
46
|
+
return field.hasMany ? kept : (kept[0] ?? undefined);
|
|
47
|
+
}
|
|
48
|
+
case "array":
|
|
49
|
+
return (Array.isArray(value) ? value : [value])
|
|
50
|
+
.map((v) => (field.of.type === "image" ? ctx.media(v) : String(v)))
|
|
51
|
+
.filter(Boolean);
|
|
52
|
+
case "json":
|
|
53
|
+
if (component === "link") {
|
|
54
|
+
// ACF post-object and relationship fields store a post id where a link is declared.
|
|
55
|
+
const id = typeof value === "object" && !value.url ? (value.ID ?? value.id) : value;
|
|
56
|
+
if (typeof id === "number" || /^\d+$/.test(String(id))) {
|
|
57
|
+
const link = ctx.refLink(id, { optional: true });
|
|
58
|
+
if (!link) ctx.warn(`${name}: post ${id} isn't imported, so its link was left out`);
|
|
59
|
+
return link ?? undefined;
|
|
60
|
+
}
|
|
61
|
+
return ctx.link(value) ?? undefined;
|
|
62
|
+
}
|
|
63
|
+
if (field.itemFields) {
|
|
64
|
+
const rows = (Array.isArray(value) ? value : []).map((row) => autoFields(field.itemFields!, row, ctx));
|
|
65
|
+
return rows.filter((row) => Object.keys(row).length);
|
|
66
|
+
}
|
|
67
|
+
if (component === "taxonomy-select") return (Array.isArray(value) ? value : [value]).map(String);
|
|
68
|
+
return value;
|
|
69
|
+
case "text":
|
|
70
|
+
if (component === "color") return String(value).toUpperCase();
|
|
71
|
+
return typeof value === "string" && /<[a-z]/i.test(value) ? ctx.text(value) : String(value);
|
|
72
|
+
case "blocks":
|
|
73
|
+
return undefined;
|
|
74
|
+
default:
|
|
75
|
+
return typeof value === "object" ? value : String(value);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/** Every declared field filled from the same-named source value (camelCase or snake_case). */
|
|
80
|
+
export const autoFields = (fields: Record<string, FieldConfig>, source: Record<string, any>, ctx: MapContext) => {
|
|
81
|
+
const out: Record<string, unknown> = {};
|
|
82
|
+
for (const [name, field] of Object.entries(fields)) {
|
|
83
|
+
const value = convertField(field, pick(source, name), ctx, name);
|
|
84
|
+
if (value !== undefined) out[name] = value;
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
/** `jobTitles`, `job-titles` and `job_title` are one taxonomy: compare without case, separators or plural. */
|
|
90
|
+
const termKey = (name: string) =>
|
|
91
|
+
name
|
|
92
|
+
.toLowerCase()
|
|
93
|
+
.replace(/[^a-z0-9]/g, "")
|
|
94
|
+
.replace(/s$/, "");
|
|
95
|
+
|
|
96
|
+
const TERM_KEYS: Record<string, string[]> = { tags: ["post_tag"], categories: ["category"], category: ["category"] };
|
|
97
|
+
|
|
98
|
+
/** Standard post columns, Yoast SEO and terms by field name; everything else from ACF meta by name. */
|
|
99
|
+
const RECORD_SOURCES: Record<string, (src: SourceRecord) => unknown> = {
|
|
100
|
+
title: (src) => src.title,
|
|
101
|
+
slug: (src) => (src.frontPage ? "home" : src.slug),
|
|
102
|
+
excerpt: (src) => src.excerpt,
|
|
103
|
+
publishedAt: (src) => src.date,
|
|
104
|
+
date: (src) => src.date,
|
|
105
|
+
parent: (src) => src.parent,
|
|
106
|
+
image: (src) => src.featuredImage,
|
|
107
|
+
featuredImage: (src) => src.featuredImage,
|
|
108
|
+
seoTitle: (src) => src.meta?._yoast_wpseo_title,
|
|
109
|
+
metaTitle: (src) => src.meta?._yoast_wpseo_title,
|
|
110
|
+
seoDescription: (src) => src.meta?._yoast_wpseo_metadesc,
|
|
111
|
+
metaDescription: (src) => src.meta?._yoast_wpseo_metadesc,
|
|
112
|
+
ogTitle: (src) => src.meta?.["_yoast_wpseo_opengraph-title"],
|
|
113
|
+
ogDescription: (src) => src.meta?.["_yoast_wpseo_opengraph-description"],
|
|
114
|
+
ogImage: (src) => src.meta?.["_yoast_wpseo_opengraph-image-id"] || src.meta?.["_yoast_wpseo_opengraph-image"],
|
|
115
|
+
noindex: (src) => src.meta?.["_yoast_wpseo_meta-robots-noindex"] === "1" || undefined,
|
|
116
|
+
nofollow: (src) => src.meta?.["_yoast_wpseo_meta-robots-nofollow"] === "1" || undefined,
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
export const autoRecord = (src: SourceRecord, collection: CollectionConfig, ctx: MapContext) => {
|
|
120
|
+
const out: Record<string, unknown> = {};
|
|
121
|
+
let body = false;
|
|
122
|
+
for (const [name, field] of Object.entries(collection.fields)) {
|
|
123
|
+
let raw: unknown;
|
|
124
|
+
if (field.type === "content" && !body) {
|
|
125
|
+
body = true;
|
|
126
|
+
raw = src.content;
|
|
127
|
+
} else if (field.admin?.component === "taxonomy-select") {
|
|
128
|
+
const taxonomy = String(field.admin.taxonomy ?? name);
|
|
129
|
+
const wanted = new Set([name, taxonomy, ...(TERM_KEYS[name] ?? []), ...(TERM_KEYS[taxonomy] ?? [])].map(termKey));
|
|
130
|
+
const key = Object.keys(src.terms ?? {}).find((k) => wanted.has(termKey(k)) && src.terms[k]?.length);
|
|
131
|
+
const slugs = key ? (src.terms[key] as any[]).map((t) => t.slug) : undefined;
|
|
132
|
+
raw = field.type === "text" ? slugs?.[0] : slugs;
|
|
133
|
+
} else raw = RECORD_SOURCES[name]?.(src) ?? pick(src.meta, name);
|
|
134
|
+
if (field.type === "relation" && name === "parent") {
|
|
135
|
+
const parent = ctx.ref(raw, { optional: true });
|
|
136
|
+
if (parent) out[name] = parent;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
const value = convertField(field, raw, ctx, name);
|
|
140
|
+
if (value !== undefined) out[name] = value;
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
};
|