@kidecms/wordpress 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/live.ts ADDED
@@ -0,0 +1,282 @@
1
+ /**
2
+ * `kide-wp check-live`: the rendered source site is the judge. Every planned document with a
3
+ * WordPress permalink is fetched from the live site; its visible content blocks must be found in
4
+ * the Kide document. Catches content a theme renders from somewhere the database checks can't
5
+ * see (template fields, options pages, widgets). Text shared by many pages (navigation, footer,
6
+ * cookie banners) and text of other imported documents (listings, teasers) is reported apart.
7
+ */
8
+ import { createHash } from "node:crypto";
9
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
10
+ import { homedir } from "node:os";
11
+ import path from "node:path";
12
+
13
+ import { decodeEntities } from "@kidecms/core";
14
+ import { htmlText, outputStrings, words } from "./domain";
15
+ import { parseBlocks, walkBlocks } from "./gutenberg";
16
+ import { loadPlan } from "./report";
17
+ import { getMeta, loadRecords, type Staging } from "./staging";
18
+
19
+ export type LivePage = {
20
+ url: string;
21
+ docId: string;
22
+ collection: string;
23
+ locale: string;
24
+ status: number;
25
+ blocks: number;
26
+ found: number;
27
+ listed: number;
28
+ /** In this page's own source record but not in its Kide document: a real loss. */
29
+ missing: string[];
30
+ /** In another staged record that isn't migrated (skipped type, filtered record). */
31
+ skipped: Array<{ text: string; source: string }>;
32
+ /** Nowhere in the dump: rendered by a plugin, feed or template, or changed after the export. */
33
+ drift: string[];
34
+ };
35
+
36
+ export type LiveReport = { pages: LivePage[]; boilerplate: number; failed: Array<{ url: string; message: string }> };
37
+
38
+ const BLOCK_SPLIT =
39
+ /<\/?(?:p|h[1-6]|li|div|section|article|blockquote|td|th|figcaption|dd|dt|br|tr|ul|ol|figure|form|label|button|a)\b[^>]*>/gi;
40
+
41
+ /** Visible text blocks of the page's main content (falls back to body without header/nav/footer). */
42
+ export const contentBlocks = (html: string): string[] => {
43
+ let body = html
44
+ .replace(/<!--[\s\S]*?-->/g, " ")
45
+ .replace(/<(script|style|noscript|svg|template|iframe)\b[\s\S]*?<\/\1>/gi, " ");
46
+ const main = /<main\b[^>]*>([\s\S]*?)<\/main>/i.exec(body);
47
+ body = main ? main[1] : body.replace(/<(header|nav|footer|aside)\b[\s\S]*?<\/\1>/gi, " ");
48
+ return body
49
+ .split(BLOCK_SPLIT)
50
+ .map((chunk) =>
51
+ decodeEntities(chunk.replace(/<[^>]+>/g, " "))
52
+ .replace(/\s+/g, " ")
53
+ .trim(),
54
+ )
55
+ .filter((text) => words(text).length >= 4);
56
+ };
57
+
58
+ const key = (text: string) => words(text).join(" ");
59
+
60
+ /** A block is found when its words appear in order in the document text, or nearly all of them do. */
61
+ const contains = (haystack: string, bag: Map<string, number>, block: string) => {
62
+ const k = key(block);
63
+ if (haystack.includes(k)) return true;
64
+ const ws = words(block);
65
+ const need = new Map<string, number>();
66
+ for (const w of ws) need.set(w, (need.get(w) ?? 0) + 1);
67
+ let hit = 0;
68
+ for (const [w, n] of need) hit += Math.min(n, bag.get(w) ?? 0);
69
+ return hit / ws.length >= 0.9;
70
+ };
71
+
72
+ const pool = async <T>(items: T[], size: number, run: (item: T) => Promise<void>) => {
73
+ let i = 0;
74
+ await Promise.all(
75
+ Array.from({ length: Math.min(size, items.length) }, async () => {
76
+ while (i < items.length) await run(items[i++]);
77
+ }),
78
+ );
79
+ };
80
+
81
+ export const checkLive = async (
82
+ staging: Staging,
83
+ options: { baseUrl?: string; concurrency?: number; limit?: number; refresh?: boolean } = {},
84
+ ): Promise<LiveReport> => {
85
+ const plan = loadPlan(staging);
86
+ if (!plan) throw new Error("No plan yet — run `kide-wp plan` first.");
87
+ const base = (options.baseUrl ?? getMeta(staging, "siteUrl") ?? "").replace(/\/$/, "");
88
+ if (!base) throw new Error("No site URL — pass --base <https://site>.");
89
+ const records = new Map(loadRecords(staging).map((r) => [r.key, r]));
90
+
91
+ // Every staged record's visible text, to tell a real loss from drift or a skipped source.
92
+ const recordText = (r: Record<string, unknown>): string => {
93
+ const parts: string[] = [];
94
+ for (const [k, v] of Object.entries(r)) {
95
+ if (k === "content" && typeof v === "string") {
96
+ parts.push(htmlText(v.replace(/<!--|-->/g, " ")));
97
+ for (const b of walkBlocks(parseBlocks(v))) parts.push(...outputStrings(b.attrs).map(htmlText));
98
+ } else parts.push(...outputStrings(v).map((t) => (/<[a-z]/i.test(t) ? htmlText(t) : t)));
99
+ }
100
+ return words(parts.join(" ")).join(" ");
101
+ };
102
+ const sourceKeys: string[] = [];
103
+ const sourceOffsets: number[] = [];
104
+ let sourceAll = "";
105
+ for (const [k, r] of records) {
106
+ sourceKeys.push(k);
107
+ sourceOffsets.push(sourceAll.length);
108
+ sourceAll += `${recordText(r)} \u0000 `;
109
+ }
110
+ const sourceOf = (k: string): string | undefined => {
111
+ const at = sourceAll.indexOf(k);
112
+ if (at < 0) return undefined;
113
+ let lo = 0;
114
+ let hi = sourceOffsets.length - 1;
115
+ while (lo < hi) {
116
+ const mid = (lo + hi + 1) >> 1;
117
+ if (sourceOffsets[mid] <= at) lo = mid;
118
+ else hi = mid - 1;
119
+ }
120
+ return sourceKeys[lo];
121
+ };
122
+ const pageSource = new Map<string, string>();
123
+ // Shared across projects and fresh runs like the media cache; `--refresh` refetches.
124
+ const cacheDir = path.resolve(process.env.KIDE_LIVE_CACHE ?? path.join(homedir(), ".cache", "kide", "import-live"));
125
+ mkdirSync(cacheDir, { recursive: true });
126
+
127
+ type Target = { url: string; docId: string; collection: string; locale: string; text: string[]; refs: string[] };
128
+ const docText = new Map(plan.docs.map((d) => [d.docId, outputStrings(d.data)]));
129
+ const targets: Target[] = [];
130
+ const known: string[] = [];
131
+ for (const doc of plan.docs) {
132
+ const baseLocale = String(doc.data._sourceLocale ?? "");
133
+ for (const k of doc.sourceKeys) {
134
+ const record = records.get(k);
135
+ const locale = String(record?.locale ?? baseLocale);
136
+ const fields =
137
+ locale === baseLocale || !doc.translations[locale] ? doc.data : { ...doc.data, ...doc.translations[locale] };
138
+ const text = outputStrings(fields);
139
+ known.push(...text.filter((t) => words(t).length >= 4));
140
+ if (record?.status !== "publish" || typeof record.permalink !== "string") continue;
141
+ // Documents this page embeds or links (a form, liftups): their text renders on it too.
142
+ const refs = [...new Set(outputStrings(fields).filter((s) => s !== doc.docId && docText.has(s)))];
143
+ targets.push({
144
+ url: `${base}${record.permalink}`,
145
+ docId: doc.docId,
146
+ collection: doc.collection,
147
+ locale,
148
+ text,
149
+ refs,
150
+ });
151
+ pageSource.set(`${base}${record.permalink}`, k);
152
+ }
153
+ }
154
+ const selected = options.limit ? targets.slice(0, options.limit) : targets;
155
+
156
+ const fetched = new Map<string, { status: number; html: string }>();
157
+ const failed: LiveReport["failed"] = [];
158
+ await pool(selected, options.concurrency ?? 6, async (t) => {
159
+ const file = path.join(cacheDir, `${createHash("sha1").update(t.url).digest("hex")}.json`);
160
+ if (!options.refresh && existsSync(file)) {
161
+ fetched.set(t.url, JSON.parse(readFileSync(file, "utf8")));
162
+ return;
163
+ }
164
+ try {
165
+ const res = await fetch(t.url, { redirect: "follow", signal: AbortSignal.timeout(30_000) });
166
+ const entry = { status: res.status, html: res.ok ? await res.text() : "" };
167
+ fetched.set(t.url, entry);
168
+ writeFileSync(file, JSON.stringify(entry));
169
+ } catch (e) {
170
+ failed.push({ url: t.url, message: (e as Error).message });
171
+ }
172
+ });
173
+
174
+ const pageBlocks = new Map<string, string[]>();
175
+ const frequency = new Map<string, number>();
176
+ for (const t of selected) {
177
+ const entry = fetched.get(t.url);
178
+ if (!entry?.html) continue;
179
+ const blocks = [...new Set(contentBlocks(entry.html))];
180
+ pageBlocks.set(t.url, blocks);
181
+ for (const b of blocks) frequency.set(key(b), (frequency.get(key(b)) ?? 0) + 1);
182
+ }
183
+ const threshold = Math.max(3, Math.ceil(pageBlocks.size * 0.05));
184
+ const boilerplate = new Set([...frequency].filter(([, n]) => n >= threshold).map(([k]) => k));
185
+ const knownText = known.map(key).join(" \u0000 ");
186
+
187
+ const pages: LivePage[] = [];
188
+ for (const t of selected) {
189
+ const entry = fetched.get(t.url);
190
+ if (!entry) continue;
191
+ const blocks = (pageBlocks.get(t.url) ?? []).filter((b) => !boilerplate.has(key(b)));
192
+ const own = t.text.map(key).join(" \u0000 ");
193
+ const bag = new Map<string, number>();
194
+ for (const w of words(t.text.join(" "))) bag.set(w, (bag.get(w) ?? 0) + 1);
195
+ const refText = t.refs.flatMap((id) => docText.get(id) ?? []);
196
+ const refBag = new Map<string, number>();
197
+ for (const w of words(refText.join(" "))) refBag.set(w, (refBag.get(w) ?? 0) + 1);
198
+ const page: LivePage = {
199
+ url: t.url,
200
+ docId: t.docId,
201
+ collection: t.collection,
202
+ locale: t.locale,
203
+ status: entry.status,
204
+ blocks: blocks.length,
205
+ found: 0,
206
+ listed: 0,
207
+ missing: [],
208
+ skipped: [],
209
+ drift: [],
210
+ };
211
+ for (const b of blocks) {
212
+ if (contains(own, bag, b)) page.found++;
213
+ else if (knownText.includes(key(b)) || (refBag.size && contains(refText.map(key).join(" "), refBag, b)))
214
+ page.listed++;
215
+ else {
216
+ const from = sourceOf(key(b));
217
+ if (!from) page.drift.push(b);
218
+ else if (from === pageSource.get(t.url)) page.missing.push(b);
219
+ else
220
+ page.skipped.push({ text: b, source: `${records.get(from)?.type} ${from} (${records.get(from)?.status})` });
221
+ }
222
+ }
223
+ pages.push(page);
224
+ }
225
+ return { pages, boilerplate: boilerplate.size, failed };
226
+ };
227
+
228
+ /** Share of the page's migratable content that reached Kide; drift and skipped sources don't count. */
229
+ export const liveScore = (p: LivePage) => {
230
+ const scope = p.found + p.listed + p.missing.length;
231
+ return scope ? (p.found + p.listed) / scope : 1;
232
+ };
233
+
234
+ export const renderLive = (report: LiveReport, minScore: number): string => {
235
+ const score = liveScore;
236
+ const byCollection = new Map<string, LivePage[]>();
237
+ for (const p of report.pages) byCollection.set(p.collection, [...(byCollection.get(p.collection) ?? []), p]);
238
+ const pct = (n: number) => `${(n * 100).toFixed(1)}%`;
239
+ const below = report.pages.filter((p) => p.status === 200 && score(p) < minScore).sort((a, b) => score(a) - score(b));
240
+ const lines = [
241
+ "# Live check",
242
+ "",
243
+ `${report.pages.length} pages compared with the live site · ${report.boilerplate} boilerplate blocks ignored · ${below.length} below ${pct(minScore)}`,
244
+ "",
245
+ "Score = share of the page's source content found in its Kide document (or, for listings, in another imported document).",
246
+ "Blocks missing from Kide are looked up in the dump: in the page's own record → **missing** (counts against the score);",
247
+ "in another staged record that isn't migrated → **skipped source**; nowhere in the dump → **not in the dump** (rendered by a plugin, feed or template, or changed after the export).",
248
+ "",
249
+ "| collection | pages | mean score | below threshold | pages with skipped-source text | pages with text not in the dump | not 200 |",
250
+ "| --- | --- | --- | --- | --- | --- | --- |",
251
+ ...[...byCollection].map(([c, ps]) => {
252
+ const ok = ps.filter((p) => p.status === 200);
253
+ const mean = ok.length ? ok.reduce((a, p) => a + score(p), 0) / ok.length : 1;
254
+ return `| \`${c}\` | ${ps.length} | ${pct(mean)} | ${ok.filter((p) => score(p) < minScore).length} | ${ok.filter((p) => p.skipped.length).length} | ${ok.filter((p) => p.drift.length).length} | ${ps.length - ok.length} |`;
255
+ }),
256
+ "",
257
+ ];
258
+ for (const p of report.pages.filter((x) => x.status !== 200)) lines.push(`- HTTP ${p.status}: ${p.url}`);
259
+ const cut = (t: string) => (t.length > 140 ? `${t.slice(0, 139)}…` : t);
260
+ for (const p of below) {
261
+ lines.push("", `## ${p.url} — ${pct(score(p))} (\`${p.collection}/${p.docId}\`, ${p.locale})`, "");
262
+ for (const m of p.missing.slice(0, 8)) lines.push(`- missing: ${cut(m)}`);
263
+ }
264
+ const skippedBySource = new Map<string, Set<string>>();
265
+ for (const p of report.pages)
266
+ for (const s of p.skipped) {
267
+ const type = s.source.split(" ")[0];
268
+ skippedBySource.set(type, (skippedBySource.get(type) ?? new Set()).add(p.url));
269
+ }
270
+ if (skippedBySource.size) {
271
+ lines.push("", "## Live text from sources that aren't migrated", "");
272
+ for (const [type, urls] of skippedBySource)
273
+ lines.push(`- \`${type}\` on ${urls.size} page(s), e.g. ${[...urls].slice(0, 3).join(", ")}`);
274
+ }
275
+ const drifted = report.pages.filter((p) => p.drift.length);
276
+ if (drifted.length) {
277
+ lines.push("", `## Not in the dump: ${drifted.length} page(s) show text the export doesn't contain`, "");
278
+ for (const p of drifted) lines.push(`- ${p.url}: ${cut(p.drift[0])}`);
279
+ }
280
+ for (const f of report.failed) lines.push(`- fetch failed: ${f.url} — ${f.message}`);
281
+ return `${lines.join("\n")}\n`;
282
+ };
package/src/mapping.ts ADDED
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Mapping files (`migration/*.map.ts`) describe source → Kide as plain
3
+ * functions. The engine runs them on proxied records and records every leaf the
4
+ * mapping reads, so coverage is derived from what the code actually does — no
5
+ * separate list of mapped fields to keep in sync.
6
+ */
7
+ import type { ContentDocument, ContentNode, RichTextDocument, StoredLink } from "@kidecms/core";
8
+ import { decodeEntities } from "@kidecms/core";
9
+ import { htmlText } from "./domain";
10
+ import type { ShortcodeHandler } from "./gutenberg";
11
+ import type { StagedRecord } from "./staging";
12
+
13
+ export type SourceRecord = StagedRecord & Record<string, any>;
14
+
15
+ export type BlockInput = {
16
+ name: string;
17
+ attrs: Record<string, any>;
18
+ /** ACF block fields with repeaters/flexible content nested (empty for non-ACF blocks). */
19
+ data: Record<string, any>;
20
+ innerHTML: string;
21
+ /** Inner blocks rendered as Kide content nodes (same rules as `ctx.content`); rendered once. */
22
+ inner: () => ContentNode[];
23
+ /** Inner blocks as raw inputs (each with `.name`), for a parent built from its children (a carousel from its cards). */
24
+ children: () => BlockInput[];
25
+ /** The declared block's fields (default: the same-named block) filled by name, to spread and override in a handler. */
26
+ auto: (blockType?: string) => Record<string, unknown>;
27
+ };
28
+
29
+ export type MapContext = {
30
+ /**
31
+ * Kide `_id` of the document a source post maps to (translations resolve to their group's
32
+ * document). A record key such as `"form:4"` addresses other sources.
33
+ */
34
+ ref: (id: unknown, options?: { optional?: boolean }) => string | null;
35
+ refs: (ids: unknown, options?: { optional?: boolean }) => string[];
36
+ /** Kide `_id` of the document a source term maps to. */
37
+ term: (id: unknown, options?: { optional?: boolean }) => string | null;
38
+ /** Source post id → link field value referencing its Kide document (survives slug changes). */
39
+ refLink: (id: unknown, options?: { optional?: boolean }) => StoredLink | null;
40
+ /** Attachment id or uploads URL → image/file value (uploaded on apply, deduped by content hash). */
41
+ media: (idOrUrl: unknown, options?: { optional?: boolean }) => string | null;
42
+ /** Gutenberg/classic HTML → `content` value; blocks go through the mapping's `blocks` handlers. */
43
+ content: (html: unknown) => ContentDocument;
44
+ /** HTML → rich text (no inline blocks). */
45
+ richText: (html: unknown) => RichTextDocument;
46
+ /** HTML → plain text. */
47
+ text: (html: unknown) => string;
48
+ /** ACF link array or URL → link field value. Internal URLs become root-relative. */
49
+ link: (value: unknown) => StoredLink | null;
50
+ /** WP / ACF date → ISO date string. */
51
+ date: (value: unknown) => string | null;
52
+ /** Another staged record, untracked (for lookups such as menu targets). */
53
+ record: (keyOrId: unknown) => SourceRecord | undefined;
54
+ /** The `locale` member of a record's translation group (a key like `"term:12"`, or the record itself), untracked. */
55
+ translation: (keyOrRecord: unknown, locale: string | undefined) => SourceRecord | undefined;
56
+ /** Records of a type listed in the mapping's `includes`, tracked: their fields join this mapping's coverage. */
57
+ records: (type: string, filter?: (record: SourceRecord) => boolean) => SourceRecord[];
58
+ /**
59
+ * Every field of the mapping's collection filled by name and declared type: title, slug (`home` for
60
+ * the front page), the content body, featured image, excerpt, date, parent, terms, Yoast SEO, and
61
+ * ACF meta of the same name (camelCase or snake_case). Spread it and override what differs.
62
+ */
63
+ auto: (src: SourceRecord) => Record<string, unknown>;
64
+ /** A nav menu term → `menu-items` tree from its `nav_menu_item` records (needs `includes: ["nav_menu_item"]`). */
65
+ menuItems: (menu: SourceRecord) => MenuItem[];
66
+ /** A `taxonomy` record → `taxonomy-terms` tree in its locale (needs `includes: ["term:<taxonomy>"]`). */
67
+ termTree: (taxonomy: SourceRecord, options?: { split?: boolean }) => TermNode[];
68
+ /** A Gravity Forms `form` record → the fields of the `forms` preset collection. */
69
+ form: (form: SourceRecord) => Record<string, unknown>;
70
+ locale: string | undefined;
71
+ warn: (message: string) => void;
72
+ };
73
+
74
+ export type MenuItem = { id: string; label: string; href: string; target?: "_blank"; children: MenuItem[] };
75
+ export type TermNode = { id: string; name: string; slug: string; children: TermNode[] };
76
+
77
+ export type BlockHandler = (
78
+ block: BlockInput,
79
+ ctx: MapContext,
80
+ ) => ContentNode | ContentNode[] | string | null | undefined;
81
+
82
+ export type Mapping = {
83
+ /** Source type: a WP post type (`page`, `post`, `case`) or `term:<taxonomy>`. */
84
+ source: string;
85
+ collection: string;
86
+ /**
87
+ * Return a reason string to leave a record out (counted in the ledger), or false. Several mappings
88
+ * may share a `source` (posts split into collections by category): each record goes to the one
89
+ * whose filter returns true, and `false` there means "not mine" rather than "filtered".
90
+ */
91
+ filter?: (src: SourceRecord) => boolean | string;
92
+ /** Kide `_id` for the document. Defaults to `wp-<id>` / `wp-term-<id>`. */
93
+ id?: (src: SourceRecord) => string;
94
+ map: (src: SourceRecord, ctx: MapContext) => Record<string, unknown>;
95
+ /**
96
+ * Source types folded into this mapping's documents (menu items into menus, terms into a
97
+ * taxonomy). Read them with `ctx.records(type)`; their paths are covered here as `<type>:<path>`.
98
+ */
99
+ includes?: string[];
100
+ /** Paths deliberately not migrated → reason. Globs allowed: `meta.legacy_*`, `block:core/spacer`. */
101
+ drop?: Record<string, string>;
102
+ blocks?: Record<string, BlockHandler>;
103
+ /** `"set"` when the mapping deliberately dedupes (term lists): each distinct source word must arrive, not every copy. */
104
+ textMatch?: "multiset" | "set";
105
+ /**
106
+ * Shortcode name → handler (HTML, or content nodes such as a form block), or "keep" to leave it as
107
+ * literal text (code samples, `[placeholder]` prose). `"*"` applies to all others.
108
+ */
109
+ shortcodes?: Record<string, ShortcodeHandler | "keep">;
110
+ };
111
+
112
+ export type Skip = { source: string; skip: string };
113
+ export type MappingModule = Mapping | Skip | Array<Mapping | Skip>;
114
+
115
+ export const defineMapping = (mapping: Mapping): Mapping => mapping;
116
+ export const skipSource = (source: string, reason: string): Skip => ({ source, skip: reason });
117
+
118
+ export const isSkip = (m: Mapping | Skip): m is Skip => "skip" in m;
119
+
120
+ // --- read tracking ------------------------------------------------------------
121
+
122
+ export class Tracker {
123
+ readonly expected: string[] = [];
124
+ private readonly strings = new Map<string, string>();
125
+ private readonly consumed = new Set<string>();
126
+
127
+ constructor(readonly reads: Set<string>) {}
128
+
129
+ read(path: string, value: unknown) {
130
+ this.reads.add(path);
131
+ if (typeof value === "string" && /\S\s+\S/.test(value)) this.strings.set(value, path);
132
+ }
133
+
134
+ /** An HTML input handed to a ctx transform: its text is expected via `expect`, not as a raw read. */
135
+ consume(value: unknown) {
136
+ if (typeof value === "string") this.consumed.add(value);
137
+ }
138
+
139
+ expect(text: string) {
140
+ if (text.trim()) this.expected.push(text);
141
+ }
142
+
143
+ expectedTexts(): string[] {
144
+ const out = [...this.expected];
145
+ for (const value of this.strings.keys()) {
146
+ if (!this.consumed.has(value)) out.push(/<[a-z!]/i.test(value) ? htmlText(value) : decodeEntities(value));
147
+ }
148
+ return out;
149
+ }
150
+ }
151
+
152
+ // `nav_menu_item:` is the root of an included record, so its fields read as `nav_menu_item:title`.
153
+ const childPath = (path: string, prop: string, isArray: boolean) =>
154
+ isArray ? `${path}[]` : !path || path.endsWith(":") ? `${path}${prop}` : `${path}.${prop}`;
155
+
156
+ export const track = <T>(value: T, path: string, tracker: Tracker): T => {
157
+ if (value === null || typeof value !== "object") return value;
158
+ const isArray = Array.isArray(value);
159
+ return new Proxy(value as object, {
160
+ get(target, prop, receiver) {
161
+ const v = Reflect.get(target, prop, receiver);
162
+ if (typeof prop === "symbol" || typeof v === "function") return v;
163
+ if (isArray && !/^\d+$/.test(prop)) return v;
164
+ const p = childPath(path, prop, isArray);
165
+ if (v !== null && typeof v === "object") return track(v, p, tracker);
166
+ if (v !== undefined && v !== null && v !== "") tracker.read(p, v);
167
+ return v;
168
+ },
169
+ }) as T;
170
+ };
@@ -0,0 +1,202 @@
1
+ /**
2
+ * Streaming mysqldump reader. Yields one `{ table, columns, rows }` per INSERT
3
+ * statement for the tables `want` accepts; every other statement is skipped
4
+ * without being buffered. Handles extended inserts, multi-line phpMyAdmin
5
+ * exports, `--complete-insert`, comments and .gz input.
6
+ */
7
+ import { createReadStream } from "node:fs";
8
+ import { createGunzip } from "node:zlib";
9
+
10
+ export type SqlValue = string | number | null;
11
+ export type InsertBatch = { table: string; columns: string[]; rows: SqlValue[][] };
12
+
13
+ const HEAD_DONE =
14
+ /^(INSERT\s+(?:IGNORE\s+)?INTO|REPLACE\s+INTO|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?)\s+`?([^`\s(]+)`?[\s(]/i;
15
+ const STATEMENT_HEAD =
16
+ /^\s*(INSERT\s+(?:IGNORE\s+)?INTO|REPLACE\s+INTO|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?)\s+`?([^`\s(]+)`?/i;
17
+
18
+ export async function* readMysqlDump(file: string, want: (table: string) => boolean): AsyncGenerator<InsertBatch> {
19
+ const raw = createReadStream(file);
20
+ const stream = file.endsWith(".gz") ? raw.pipe(createGunzip()) : raw;
21
+ stream.setEncoding("utf8");
22
+
23
+ const columns = new Map<string, string[]>();
24
+ let buf: string[] = [];
25
+ let head = "";
26
+ let mode: "head" | "keep" | "skip" = "head";
27
+ let quote = "";
28
+ let escape = false;
29
+ let lineComment = false;
30
+ let blockComment = false;
31
+ let prev = "";
32
+
33
+ const finish = (): InsertBatch | null => {
34
+ const text = buf.join("");
35
+ buf = [];
36
+ const keep = mode === "keep";
37
+ mode = "head";
38
+ head = "";
39
+ if (!keep) return null;
40
+ const m = STATEMENT_HEAD.exec(text);
41
+ if (!m) return null;
42
+ const table = m[2];
43
+ if (/^CREATE/i.test(m[1])) {
44
+ columns.set(table, parseCreateColumns(text));
45
+ return null;
46
+ }
47
+ const parsed = parseInsert(text, m.index + m[0].length);
48
+ return { table, columns: parsed.columns ?? columns.get(table) ?? [], rows: parsed.rows };
49
+ };
50
+
51
+ for await (const chunk of stream as AsyncIterable<string>) {
52
+ let start = 0;
53
+ for (let i = 0; i < chunk.length; i++) {
54
+ const c = chunk[i];
55
+ if (lineComment) {
56
+ if (c === "\n") {
57
+ lineComment = false;
58
+ buf = [];
59
+ start = i + 1;
60
+ }
61
+ continue;
62
+ }
63
+ if (blockComment) {
64
+ if (prev === "*" && c === "/") {
65
+ blockComment = false;
66
+ buf = [];
67
+ start = i + 1;
68
+ }
69
+ prev = c;
70
+ continue;
71
+ }
72
+ if (mode === "head" && !(head === "" && /\s/.test(c))) {
73
+ head += c;
74
+ if (head === "--" || head === "#") {
75
+ lineComment = true;
76
+ head = "";
77
+ continue;
78
+ }
79
+ if (head === "/*") {
80
+ blockComment = true;
81
+ head = "";
82
+ prev = "";
83
+ continue;
84
+ }
85
+ const m = HEAD_DONE.exec(head);
86
+ if (m && !/^IF$/i.test(m[2])) mode = want(m[2]) ? "keep" : "skip";
87
+ else if ((head.length >= 8 && !/^(INSERT|REPLACE|CREATE)/i.test(head)) || head.length > 200) mode = "skip";
88
+ }
89
+ if (quote) {
90
+ if (escape) escape = false;
91
+ else if (c === "\\" && quote !== "`") escape = true;
92
+ else if (c === quote) quote = "";
93
+ } else if (c === "'" || c === '"' || c === "`") quote = c;
94
+ else if (c === ";") {
95
+ if (mode !== "skip") buf.push(chunk.slice(start, i));
96
+ start = i + 1;
97
+ const batch = finish();
98
+ if (batch) yield batch;
99
+ }
100
+ prev = c;
101
+ }
102
+ if (mode !== "skip" && !lineComment && !blockComment) buf.push(chunk.slice(start));
103
+ }
104
+ if (buf.join("").trim()) {
105
+ const batch = finish();
106
+ if (batch) yield batch;
107
+ }
108
+ }
109
+
110
+ const parseCreateColumns = (sql: string): string[] => {
111
+ const body = sql.slice(sql.indexOf("(") + 1);
112
+ const cols: string[] = [];
113
+ for (const line of body.split("\n")) {
114
+ const m = /^\s*`([^`]+)`\s+\w/.exec(line);
115
+ if (m) cols.push(m[1]);
116
+ }
117
+ return cols;
118
+ };
119
+
120
+ const ESCAPES: Record<string, string> = { "0": "\0", b: "\b", n: "\n", r: "\r", t: "\t", Z: "\x1a" };
121
+
122
+ const parseInsert = (sql: string, from: number): { columns?: string[]; rows: SqlValue[][] } => {
123
+ let i = from;
124
+ let columns: string[] | undefined;
125
+ const skipWs = () => {
126
+ while (i < sql.length && /\s/.test(sql[i])) i++;
127
+ };
128
+ skipWs();
129
+ if (sql[i] === "(") {
130
+ const end = sql.indexOf(")", i);
131
+ columns = sql
132
+ .slice(i + 1, end)
133
+ .split(",")
134
+ .map((c) => c.trim().replace(/^`|`$/g, ""));
135
+ i = end + 1;
136
+ }
137
+ const values = /VALUES/i.exec(sql.slice(i, i + 64));
138
+ if (!values) return { columns, rows: [] };
139
+ i += values.index + values[0].length;
140
+
141
+ const rows: SqlValue[][] = [];
142
+ while (i < sql.length) {
143
+ skipWs();
144
+ if (sql[i] !== "(") break;
145
+ i++;
146
+ const row: SqlValue[] = [];
147
+ while (i < sql.length) {
148
+ skipWs();
149
+ const c = sql[i];
150
+ if (c === "'" || c === '"') {
151
+ const q = c;
152
+ i++;
153
+ let out = "";
154
+ let seg = i;
155
+ while (i < sql.length) {
156
+ const ch = sql[i];
157
+ if (ch === "\\") {
158
+ out += sql.slice(seg, i);
159
+ const next = sql[i + 1];
160
+ out += ESCAPES[next] ?? next;
161
+ i += 2;
162
+ seg = i;
163
+ } else if (ch === q) {
164
+ if (sql[i + 1] === q) {
165
+ out += sql.slice(seg, i + 1);
166
+ i += 2;
167
+ seg = i;
168
+ } else break;
169
+ } else i++;
170
+ }
171
+ out += sql.slice(seg, i);
172
+ i++;
173
+ row.push(out);
174
+ } else {
175
+ let j = i;
176
+ while (j < sql.length && sql[j] !== "," && sql[j] !== ")") j++;
177
+ const token = sql.slice(i, j).trim();
178
+ i = j;
179
+ if (/^NULL$/i.test(token)) row.push(null);
180
+ else if (/^_binary\s*'/i.test(token)) row.push(token.replace(/^_binary\s*'|'$/gi, ""));
181
+ else if (/^0x[0-9a-f]+$/i.test(token)) row.push(Buffer.from(token.slice(2), "hex").toString("utf8"));
182
+ else {
183
+ const n = Number(token);
184
+ row.push(token !== "" && Number.isFinite(n) ? n : token);
185
+ }
186
+ }
187
+ skipWs();
188
+ if (sql[i] === ",") {
189
+ i++;
190
+ continue;
191
+ }
192
+ if (sql[i] === ")") {
193
+ i++;
194
+ break;
195
+ }
196
+ }
197
+ rows.push(row);
198
+ skipWs();
199
+ if (sql[i] === ",") i++;
200
+ }
201
+ return { columns, rows };
202
+ };