@contentrain/wp-import 0.12.2 → 0.13.0

package/README.md

@@ -37,6 +37,7 @@ const commentsExport = buildCommentsExport(raw, entry_source_map)
 - **UTC dates** — GMT columns preferred, stamped ISO 8601 `Z`.
 - **Pure conversion** — `rawToContentrain` returns a canonical-serialized file map (sorted keys, 2-space indent, trailing newline); writing to disk is the caller's one line.
 - **EntrySourceMap produced at the only place that can know it** — the WP-id → entry-address mapping the comments intake requires.
+- **No e-mail addresses by default** — a WXR export carries author and commenter e-mails; `rawToContentrain` leaves them out of the store unless called with `{ includeEmails: true }` (the store usually ends up in git), and `buildCommentsExport` sets `email: null` unless given the same option. Comment meta under a personal-data key (IP, user agent, e-mail, avatar URLs, every `akismet_*`; `COMMENT_PII_META_KEY`) never travels, with or without it. The REST path never has them. Names, logins, commenter website URLs and comment text are written.
 - **Only public discussion leaves the site** — `buildCommentsExport` carries comments on published, unprotected entries that are approved or pending (pending lands in the receiver's moderation queue). Comments on drafts, private, scheduled or password-protected entries, any other status (spam, trash, `post-trashed`, a plugin's) and entries the import does not hold, stay behind; `excluded` counts them.
 
 Streaming WXR parse (sax): a 100 MB export holds only its records in memory.

package/dist/index.d.mts

@@ -148,9 +148,19 @@ interface ContentrainResult {
   entry_source_map: EntrySourceMap;
   report: ImportReport;
 }
-declare function rawToContentrain(raw: RawIR, opts?: {
+interface RawToContentrainOptions {
+  /** `updated_by` stamped on every meta entry. Default `@contentrain/wp-import`. */
   updatedBy?: string;
-}): ContentrainResult;
+  /**
+   * Write author and commenter e-mail addresses into the `authors` and
+   * `comments` entries. Default `false`: a WXR export carries them, but they
+   * are third parties' personal data and a store usually ends up in git
+   * (sometimes a public repo), so they stay out unless asked for. The REST
+   * path never has them.
+   */
+  includeEmails?: boolean;
+}
+declare function rawToContentrain(raw: RawIR, opts?: RawToContentrainOptions): ContentrainResult;
 //#endregion
 //#region src/comments.d.ts
 /**
@@ -163,9 +173,26 @@ declare function selectComments(raw: RawIR): {
   comments: RawComment[];
   excluded: NonNullable<CommentsExport['excluded']>;
 };
-declare function buildCommentsExport(raw: RawIR, entries: EntrySourceMap, opts?: {
+/**
+ * Comment meta keys that hold a commenter's personal data: IP, browser user
+ * agent, e-mail, avatar URLs (a Gravatar URL carries the e-mail's hash). Same
+ * list as Migrate's intake (`PII_META_KEY`), plus every `akismet_*` key:
+ * `akismet_as_submitted` stores the whole submission (IP, user agent, e-mail,
+ * referrer) and the rest is spam-check bookkeeping. Matched by key, so a
+ * comment with no e-mail still loses its IP and user agent.
+ */
+declare const COMMENT_PII_META_KEY: RegExp;
+interface CommentsExportOptions {
   generated_at?: string;
-}): CommentsExport;
+  /**
+   * Carry commenter e-mail addresses. Default `false`: the export is written
+   * next to the store and is personal data of third parties; a receiving
+   * service that needs addresses asks for them. IPs, user agents and avatar
+   * URLs are never carried, with or without this option.
+   */
+  includeEmails?: boolean;
+}
+declare function buildCommentsExport(raw: RawIR, entries: EntrySourceMap, opts?: CommentsExportOptions): CommentsExport;
 /** Handoff summary for the export: counts + closed threads + what could not be addressed. */
 declare function summarizeComments(exp: CommentsExport): HandoffComments;
 //#endregion
@@ -266,5 +293,5 @@ declare function planSourceDelta(input: PlanSourceDeltaInput): SourceDeltaPlan;
 /** The plan as text, for a dry run: one line per record, then conflicts, unmapped records, redirects and warnings. */
 declare function formatSourceDeltaReport(plan: SourceDeltaPlan): string;
 //#endregion
-export { type ContentrainResult, DEFAULT_IMPORTERS, DEFAULT_TAXONOMIES, type DeltaStore, type ImportReport, LANDING_PAGE_TYPE, PLUGIN_META, type PlanSourceDeltaInput, type RestImportOptions, type RestImportResult, SKIP_TYPES, type UnserializeResult, type WxrParseResult, type WxrStats, addressSlug, buildCommentsExport, canon, fetchRestRawIR, formatSourceDeltaReport, hexId, looksSerialized, pageTypeOf, parseWxr, phpUnserialize, planSourceDelta, rawToContentrain, selectComments, slugify, strip, summarizeComments, taxModelId, tryUnserialize };
+export { COMMENT_PII_META_KEY, type CommentsExportOptions, type ContentrainResult, DEFAULT_IMPORTERS, DEFAULT_TAXONOMIES, type DeltaStore, type ImportReport, LANDING_PAGE_TYPE, PLUGIN_META, type PlanSourceDeltaInput, type RawToContentrainOptions, type RestImportOptions, type RestImportResult, SKIP_TYPES, type UnserializeResult, type WxrParseResult, type WxrStats, addressSlug, buildCommentsExport, canon, fetchRestRawIR, formatSourceDeltaReport, hexId, looksSerialized, pageTypeOf, parseWxr, phpUnserialize, planSourceDelta, rawToContentrain, selectComments, slugify, strip, summarizeComments, taxModelId, tryUnserialize };
 //# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../src/wxr.ts","../src/rest.ts","../src/contentrain.ts","../src/comments.ts","../src/php-unserialize.ts","../src/core.ts","../src/delta.ts"],"mappings":";;;;UA2BiB,QAAA;EACf,UAAA;IAAc,QAAA;IAAkB,OAAA;IAAiB,MAAA;IAAgB,IAAA;EAAA;EACjE,UAAA;EACA,gBAAA;EACA,MAAA,EAAQ,MAAA;AAAA;AAAA,UAGO,cAAA;EACf,GAAA,EAAK,KAAA;EACL,KAAA,EAAO,QAAA;AAAA;AAAA,iBAiBa,QAAA,CAAS,KAAA,WAAgB,QAAA,EAAU,IAAA;EAAS,IAAA;AAAA,IAAkB,OAAA,CAAQ,cAAA;;;UCnC3E,iBAAA;EACf,MAAA;;EAEA,SAAA,UAAmB,KAAA;EDMI;;;;;;;;ECGvB,IAAA;IAAS,IAAA;IAAc,WAAA;EAAA;EACvB,OAAA;EDAc;AAGhB;;;;ECGE,WAAA;EDFK;;;;ECOL,QAAA;EACA,IAAA;AAAA;AAAA,UAGe,gBAAA;EACf,GAAA,EAAK,KAAA;EACL,QAAA;EDKkF;;;;;;;;ECIlF,UAAA;IAAc,MAAA;IAA0C,SAAA;EAAA;;;AAvC1D;;;;;;;;;;EAoDE,IAAA;AAAA;AAAA,iBA4EoB,cAAA,CAAe,OAAA,EAAS,iBAAA,GAAoB,OAAA,CAAQ,gBAAA;;;;UCnHzD,QAAA;EACf,KAAA;EACA,IAAA;EFNuB;EEQvB,IAAA;EFJc;EEMd,IAAA;EFTc;;;;EEcd,MAAA;AAAA;AAAA,UAkBe,YAAA;EACf,cAAA;EACA,aAAA;EF/Bc;EEiCd,UAAA,EAAY,QAAA;EACZ,cAAA;EACA,WAAA,EAAa,MAAA;EACb,UAAA,EAAY,MAAA;EFhCZ;EEkCA,WAAA,EAAa,MAAA;EFjCb;EEmCA,mBAAA,EAAqB,MAAA;EFnCN;EEqCf,mBAAA,EAAqB,MAAA;EFpBD;EEsBpB,oBAAA;;EAEA,iBAAA,EAAmB,MAAA;EFxBuE;EE0B1F,sBAAA,EAAwB,MAAA;EACxB,aAAA;EACA,iBAAA;EACA,MAAA,EAAQ,MAAA;IAAiB,IAAA;IAAc,MAAA;IAAgB,MAAA;IAAgB,OAAA;EAAA;EF7BiC;EE+BxG,OAAA;;EAEA,kBAAA;;ADpEF;;;;;EC2EE,yBAAA;EDxEmB;;;;EC6EnB,kBAAA;AAAA;AAAA,UAGe,iBAAA;EACf,KAAA,EAAO,MAAA;EACP,gBAAA,EAAkB,cAAA;EAClB,MAAA,EAAQ,YAAA;AAAA;AAAA,iBA0CM,gBAAA,CAAiB,GAAA,EAAK,KAAA,EAAO,IAAA;EAAS,SAAA;AAAA,IAAuB,iBAAA;;;;;;AFvH7E;;;iBGbgB,cAAA,CAAe,GAAA,EAAK,KAAA;EAAU,QAAA,EAAU,UAAA;EAAc,QAAA,EAAU,WAAA,CAAY,cAAA;AAAA;AAAA,iBAgB5E,mBAAA,CAAoB,GAAA,EAAK,KAAA,EAAO,OAAA,EAAS,cAAA,EAAgB,IAAA;EAAS,YAAA;AAAA,IAA0B,cAAA;;iBAiB5F,iBAAA,CAAkB,GAAA,EAAK,cAAA,GAAiB,eAAA;;;UC3CvC,iBAAA;EACf,EAAA;EACA,UAAA;EACA,KAAA;EACA,KAAA;AAAA;AAAA,iBAGc,eAAA,CAAgB,CAAA,YAAa,CAAA;AAAA,iBAI7B,cAAA,CAAe,GAAA;;iBAyFf,cAAA,CAAe,CAAA,WAAY,iBAAA;;;;;;;;cCrE9B,KAAA,GAAS,CAAA;;cAGT,KAAA,GAAS,CAAA;;cAcT,KAAA,GAAS,CAAA;;;;;;;;ALCtB;;cK8Ba,UAAA,EAAU,MAAA;;;;;;;cASV,iBAAA;AAAA,cACA,UAAA,GAAc,IAAA;;cAGd,WAAA,EAAW,MAAA;;cAOX,UAAA,GAAc,CAAA;AAAA,cAGd,OAAA,GAAW,CAAA;;AJxFxB;;;;;cIsGa,WAAA,GAAe,GAAA;;;;UCrGX,UAAA;EACf,KAAA,EAAO,MAAA;EACP,gBAAA,EAAkB,cAAA;AAAA;AAAA,UAGH,oBAAA;ENOD;EMLd,KAAA,EAAO,eAAA;ENEO;EMAd,KAAA,EAAO,UAAA;ENA0C;EMEjD,QAAA,GAAW,UAAA;ENDX;;;;;EMOA,UAAA;ENFe;;;;EMOf,SAAA;AAAA;AAAA,cAGW,kBAAA;AAAA,cACA,iBAAA;;;ANQb;;;;;;;;;;iBM8NgB,eAAA,CAAgB,KAAA,EAAO,oBAAA,GAAuB,eAAA;;iBAkC9C,uBAAA,CAAwB,IAAA,EAAM,eAAA"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/wxr.ts","../src/rest.ts","../src/contentrain.ts","../src/comments.ts","../src/php-unserialize.ts","../src/core.ts","../src/delta.ts"],"mappings":";;;;UA2BiB,QAAA;EACf,UAAA;IAAc,QAAA;IAAkB,OAAA;IAAiB,MAAA;IAAgB,IAAA;EAAA;EACjE,UAAA;EACA,gBAAA;EACA,MAAA,EAAQ,MAAA;AAAA;AAAA,UAGO,cAAA;EACf,GAAA,EAAK,KAAA;EACL,KAAA,EAAO,QAAA;AAAA;AAAA,iBAiBa,QAAA,CAAS,KAAA,WAAgB,QAAA,EAAU,IAAA;EAAS,IAAA;AAAA,IAAkB,OAAA,CAAQ,cAAA;;;UCnC3E,iBAAA;EACf,MAAA;;EAEA,SAAA,UAAmB,KAAA;EDMI;;;;;;;;ECGvB,IAAA;IAAS,IAAA;IAAc,WAAA;EAAA;EACvB,OAAA;EDAc;AAGhB;;;;ECGE,WAAA;EDFK;;;;ECOL,QAAA;EACA,IAAA;AAAA;AAAA,UAGe,gBAAA;EACf,GAAA,EAAK,KAAA;EACL,QAAA;EDKkF;;;;;;;;ECIlF,UAAA;IAAc,MAAA;IAA0C,SAAA;EAAA;;;AAvC1D;;;;;;;;;;EAoDE,IAAA;AAAA;AAAA,iBA4EoB,cAAA,CAAe,OAAA,EAAS,iBAAA,GAAoB,OAAA,CAAQ,gBAAA;;;;UCnHzD,QAAA;EACf,KAAA;EACA,IAAA;EFNuB;EEQvB,IAAA;EFJc;EEMd,IAAA;EFTc;;;;EEcd,MAAA;AAAA;AAAA,UAkBe,YAAA;EACf,cAAA;EACA,aAAA;EF/Bc;EEiCd,UAAA,EAAY,QAAA;EACZ,cAAA;EACA,WAAA,EAAa,MAAA;EACb,UAAA,EAAY,MAAA;EFhCZ;EEkCA,WAAA,EAAa,MAAA;EFjCb;EEmCA,mBAAA,EAAqB,MAAA;EFnCN;EEqCf,mBAAA,EAAqB,MAAA;EFpBD;EEsBpB,oBAAA;;EAEA,iBAAA,EAAmB,MAAA;EFxBuE;EE0B1F,sBAAA,EAAwB,MAAA;EACxB,aAAA;EACA,iBAAA;EACA,MAAA,EAAQ,MAAA;IAAiB,IAAA;IAAc,MAAA;IAAgB,MAAA;IAAgB,OAAA;EAAA;EF7BiC;EE+BxG,OAAA;;EAEA,kBAAA;;ADpEF;;;;;EC2EE,yBAAA;EDxEmB;;;;EC6EnB,kBAAA;AAAA;AAAA,UAGe,iBAAA;EACf,KAAA,EAAO,MAAA;EACP,gBAAA,EAAkB,cAAA;EAClB,MAAA,EAAQ,YAAA;AAAA;AAAA,UA0CO,uBAAA;;EAEf,SAAA;EDrGA;;;;;;;EC6GA,aAAA;AAAA;AAAA,iBAGc,gBAAA,CAAiB,GAAA,EAAK,KAAA,EAAO,IAAA,GAAO,uBAAA,GAA0B,iBAAA;;;;;;AFpI9E;;;iBGbgB,cAAA,CAAe,GAAA,EAAK,KAAA;EAAU,QAAA,EAAU,UAAA;EAAc,QAAA,EAAU,WAAA,CAAY,cAAA;AAAA;;;;;;;;;cAwB/E,oBAAA,EAAoB,MAAA;AAAA,UAuBhB,qBAAA;EACf,YAAA;EH1Be;;;;;;EGiCf,aAAA;AAAA;AAAA,iBAGc,mBAAA,CAAoB,GAAA,EAAK,KAAA,EAAO,OAAA,EAAS,cAAA,EAAgB,IAAA,GAAO,qBAAA,GAAwB,cAAA;;iBAoBxF,iBAAA,CAAkB,GAAA,EAAK,cAAA,GAAiB,eAAA;;;UCxFvC,iBAAA;EACf,EAAA;EACA,UAAA;EACA,KAAA;EACA,KAAA;AAAA;AAAA,iBAGc,eAAA,CAAgB,CAAA,YAAa,CAAA;AAAA,iBAI7B,cAAA,CAAe,GAAA;;iBAyFf,cAAA,CAAe,CAAA,WAAY,iBAAA;;;;;;;;cCrE9B,KAAA,GAAS,CAAA;;cAGT,KAAA,GAAS,CAAA;;cAcT,KAAA,GAAS,CAAA;;;;;;;;ALCtB;;cK8Ba,UAAA,EAAU,MAAA;;;;;;;cASV,iBAAA;AAAA,cACA,UAAA,GAAc,IAAA;;cAGd,WAAA,EAAW,MAAA;;cAOX,UAAA,GAAc,CAAA;AAAA,cAGd,OAAA,GAAW,CAAA;;AJxFxB;;;;;cIsGa,WAAA,GAAe,GAAA;;;;UCrGX,UAAA;EACf,KAAA,EAAO,MAAA;EACP,gBAAA,EAAkB,cAAA;AAAA;AAAA,UAGH,oBAAA;ENOD;EMLd,KAAA,EAAO,eAAA;ENEO;EMAd,KAAA,EAAO,UAAA;ENA0C;EMEjD,QAAA,GAAW,UAAA;ENDX;;;;;EMOA,UAAA;ENFe;;;;EMOf,SAAA;AAAA;AAAA,cAGW,kBAAA;AAAA,cACA,iBAAA;;;ANQb;;;;;;;;;;iBM8NgB,eAAA,CAAgB,KAAA,EAAO,oBAAA,GAAuB,eAAA;;iBAkC9C,uBAAA,CAAwB,IAAA,EAAM,eAAA"}

package/dist/index.mjs

@@ -2200,6 +2200,7 @@ function relationOver(targets) {
 const idsOf = (v) => (Array.isArray(v) ? v : [v]).map((x) => typeof x === "number" ? x : typeof x === "string" && /^\d+$/.test(x) ? Number(x) : x && typeof x === "object" ? Number(x.ID ?? x.id ?? x.term_id) : NaN).filter((n) => Number.isSafeInteger(n) && n > 0);
 function rawToContentrain(raw, opts) {
 	const updatedBy = opts?.updatedBy ?? "@contentrain/wp-import";
+	const includeEmails = opts?.includeEmails === true;
 	const locale = normLocale(raw.site.language) || "en";
 	const models = {};
 	const contents = {};
@@ -2502,7 +2503,7 @@ function rawToContentrain(raw, opts) {
 		contents.authors[id] = pick({
 			name: strip(a.display_name),
 			slug: slugify(a.login),
-			email: a.email,
+			email: includeEmails ? a.email : null,
 			wp_id: a.id
 		}, new Set([
 			"name",
@@ -3118,7 +3119,7 @@ function rawToContentrain(raw, opts) {
 			}
 			contents.comments[cRef(c.id)] = pick({
 				author: strip(c.author) || "anonymous",
-				email: c.email,
+				email: includeEmails ? c.email : null,
 				url: c.url,
 				body: c.content,
 				published_at: c.date,
@@ -3321,9 +3322,42 @@ function selectComments(raw) {
 		excluded
 	};
 }
+/**
+* Comment meta keys that hold a commenter's personal data: IP, browser user
+* agent, e-mail, avatar URLs (a Gravatar URL carries the e-mail's hash). Same
+* list as Migrate's intake (`PII_META_KEY`), plus every `akismet_*` key:
+* `akismet_as_submitted` stores the whole submission (IP, user agent, e-mail,
+* referrer) and the rest is spam-check bookkeeping. Matched by key, so a
+* comment with no e-mail still loses its IP and user agent.
+*/
+const COMMENT_PII_META_KEY = /(?:^|[_-])(?:ip|ip_address|author_ip|user_ip|agent|user_agent|author_agent|email|author_email|mail|avatar|avatar_url|avatar_urls|author_avatar_urls)$|^_?akismet_/i;
+/** Anything shaped like an e-mail address, anywhere in a serialized value. */
+const EMAIL_RE = /[^\s@"'<>;:]+@[^\s@"'<>;:]+\.[a-z]{2,}/i;
+const serialized = (v) => typeof v === "string" ? v : JSON.stringify(v) ?? "";
+/**
+* A comment as it may leave the site. Meta under a personal-data key is always
+* dropped: IPs and user agents are never exported. Without `includeEmails`,
+* `email` becomes null and any remaining meta whose value holds an address is
+* dropped too (an unknown plugin's own key).
+*/
+function exportableComment(c, includeEmails) {
+	const out = includeEmails ? { ...c } : {
+		...c,
+		email: null
+	};
+	if (c.meta) {
+		const kept = Object.entries(c.meta).filter(([k, v]) => !COMMENT_PII_META_KEY.test(k) && (includeEmails || !EMAIL_RE.test(serialized(v))));
+		if (kept.length) out.meta = Object.fromEntries(kept);
+		else delete out.meta;
+	}
+	return out;
+}
 function buildCommentsExport(raw, entries, opts) {
 	const threadsClosed = raw.posts.filter((p) => p.comment_status && p.comment_status !== "open").map((p) => p.id);
-	const { comments, excluded } = selectComments(raw);
+	const selected = selectComments(raw);
+	const excluded = selected.excluded;
+	const includeEmails = opts?.includeEmails === true;
+	const comments = selected.comments.map((c) => exportableComment(c, includeEmails));
 	return {
 		version: MIGRATION_CONTRACT_VERSION,
 		format: COMMENTS_EXPORT_FORMAT,
@@ -3611,6 +3645,6 @@ function formatSourceDeltaReport(plan) {
 	return `${lines.join("\n")}\n`;
 }
 //#endregion
-export { DEFAULT_IMPORTERS, DEFAULT_TAXONOMIES, LANDING_PAGE_TYPE, PLUGIN_META, SKIP_TYPES, addressSlug, buildCommentsExport, canon, fetchRestRawIR, formatSourceDeltaReport, hexId, looksSerialized, pageTypeOf, parseWxr, phpUnserialize, planSourceDelta, rawToContentrain, selectComments, slugify, strip, summarizeComments, taxModelId, tryUnserialize };
+export { COMMENT_PII_META_KEY, DEFAULT_IMPORTERS, DEFAULT_TAXONOMIES, LANDING_PAGE_TYPE, PLUGIN_META, SKIP_TYPES, addressSlug, buildCommentsExport, canon, fetchRestRawIR, formatSourceDeltaReport, hexId, looksSerialized, pageTypeOf, parseWxr, phpUnserialize, planSourceDelta, rawToContentrain, selectComments, slugify, strip, summarizeComments, taxModelId, tryUnserialize };
 
 //# sourceMappingURL=index.mjs.map