ghost 6.53.0 → 6.54.1

package/core/server/services/members/import-export/csv/parse.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = parse;
+const node_stream_1 = require("node:stream");
+const papaparse_1 = __importDefault(require("papaparse"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+// A column named after an Object.prototype member (toString, __proto__, ...) is unsafe
+// as a key: writing it would shadow a method or reach the prototype.
+function isSafeColumnName(name) {
+    return !(name in Object.prototype);
+}
+// headerMapping renames the CSV's headers to the columns they emit under; unmapped
+// columns carry through under their own name.
+function parse(path, headerMapping) {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs_extra_1.default.createReadStream(path);
+        const csvParserStream = papaparse_1.default.parse(papaparse_1.default.NODE_STREAM_INPUT, {
+            header: true
+        });
+        const rows = [];
+        const parsedCSVStream = (0, node_stream_1.pipeline)(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+        parsedCSVStream.on('data', (parsedRow) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row = {};
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+                    // hasOwn, not `in`: a prototype-named header would otherwise match an
+                    // inherited method on the mapping and take a function as its mapped name.
+                    if (headerMapping && Object.hasOwn(headerMapping, header)) {
+                        row[headerMapping[header]] = value;
+                    }
+                    else if (isSafeColumnName(header)) {
+                        // Carry any unmapped column through untouched, so the import is not
+                        // constrained to a known vocabulary: a custom_fields.* column
+                        // survives parsing even though nothing consumes it yet.
+                        row[header] = value;
+                    }
+                }
+                if (!Object.keys(row).length) {
+                    return;
+                }
+                rows.push(row);
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/import-export/csv/parse.ts

@@ -0,0 +1,69 @@
+import {pipeline} from 'node:stream';
+import papaparse from 'papaparse';
+import fs from 'fs-extra';
+
+// A parsed CSV row: raw string cells, keyed by (renamed) column. Parsing is mechanical;
+// giving the columns meaning and coercing their values is the domain's job (the import
+// schema).
+export type Row = Record<string, string>;
+
+// A column named after an Object.prototype member (toString, __proto__, ...) is unsafe
+// as a key: writing it would shadow a method or reach the prototype.
+function isSafeColumnName(name: string): boolean {
+    return !(name in Object.prototype);
+}
+
+// headerMapping renames the CSV's headers to the columns they emit under; unmapped
+// columns carry through under their own name.
+export default function parse(path: string, headerMapping?: Record<string, string>): Promise<Row[]> {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs.createReadStream(path);
+        const csvParserStream = papaparse.parse(papaparse.NODE_STREAM_INPUT, {
+            header: true
+        });
+
+        const rows: Row[] = [];
+        const parsedCSVStream = pipeline(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+
+        parsedCSVStream.on('data', (parsedRow: Record<string, unknown>) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row: Row = {};
+
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+
+                    // hasOwn, not `in`: a prototype-named header would otherwise match an
+                    // inherited method on the mapping and take a function as its mapped name.
+                    if (headerMapping && Object.hasOwn(headerMapping, header)) {
+                        row[headerMapping[header]] = value;
+                    } else if (isSafeColumnName(header)) {
+                        // Carry any unmapped column through untouched, so the import is not
+                        // constrained to a known vocabulary: a custom_fields.* column
+                        // survives parsing even though nothing consumes it yet.
+                        row[header] = value;
+                    }
+                }
+
+                if (!Object.keys(row).length) {
+                    return;
+                }
+
+                rows.push(row);
+            } catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/import-export/csv/serialize.js

@@ -0,0 +1,13 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = serialize;
+const papaparse_1 = __importDefault(require("papaparse"));
+// The papaparse boundary: given rows already shaped into cells, it writes CSV and
+// nothing else. escapeFormulae is forced on -- papaparse defaults it off, but this CSV
+// is opened by a person, so a leading =, +, -, @ or tab must be escaped.
+function serialize(rows, { columns, header = true } = {}) {
+    return papaparse_1.default.unparse(rows, { columns, header, escapeFormulae: true });
+}

package/core/server/services/members/import-export/csv/serialize.ts

@@ -0,0 +1,18 @@
+import papaparse from 'papaparse';
+
+interface SerializeOptions {
+    // Omitted, papaparse derives the columns (and their order) from the first row's
+    // keys -- so a typed shaper's row can define them, with no separate list to drift.
+    columns?: string[];
+    header?: boolean;
+}
+
+// The papaparse boundary: given rows already shaped into cells, it writes CSV and
+// nothing else. escapeFormulae is forced on -- papaparse defaults it off, but this CSV
+// is opened by a person, so a leading =, +, -, @ or tab must be escaped.
+export default function serialize(
+    rows: Array<Record<string, unknown>>,
+    {columns, header = true}: SerializeOptions = {}
+): string {
+    return papaparse.unparse(rows, {columns, header, escapeFormulae: true});
+}

package/core/server/services/members/import-export/export/exporter.js

@@ -0,0 +1,202 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toExportCsvRow = toExportCsvRow;
+const moment_1 = __importDefault(require("moment"));
+const node_stream_1 = require("node:stream");
+const logging = require('@tryghost/logging');
+const errors = require('@tryghost/errors');
+const { csvCellsForFields } = require('@tryghost/custom-field-types/csv');
+// The columns the export reads off the members table, checked against MemberDbRow so the
+// query cannot select a column the row type does not declare.
+const MEMBER_COLUMNS = ['id', 'email', 'name', 'note', 'status', 'created_at'];
+function namesToCsv(items) {
+    return items.map(item => item.name).join(',');
+}
+// Encode a flattened export row into its CSV cells, type-checked against MemberExportRow:
+// a change to that row must be handled here or this stops compiling. deleted_at is always
+// null -- the export reads live members -- and complimentary_plan is written from comped.
+function toExportCsvRow(row) {
+    return {
+        id: row.id,
+        email: row.email,
+        name: row.name,
+        note: row.note,
+        subscribed_to_emails: row.subscribed === true ? 'true' : 'false',
+        complimentary_plan: row.comped === true ? 'true' : '',
+        stripe_customer_id: row.stripe_customer_id,
+        created_at: row.created_at,
+        deleted_at: null,
+        labels: namesToCsv(row.labels),
+        tiers: namesToCsv(row.tiers),
+        gift_id: row.gift_id || '',
+        ...row.custom_field_cells
+    };
+}
+const BATCH_SIZE = 1000;
+// Streams the matching members as flat rows, each carrying its related data (tiers,
+// labels, subscription, gift, Stripe customer, custom fields) for the output serializer
+// to turn into CSV lines. The members aggregate and custom fields are reached only
+// through the ports above, so nothing Bookshelf- or flag-shaped leaks into the export.
+class MembersCSVExporter {
+    _knex;
+    _members;
+    _customFields;
+    constructor({ knex, members, customFields }) {
+        this._knex = knex;
+        this._members = members;
+        this._customFields = customFields;
+    }
+    async export(options = {}) {
+        const start = Date.now();
+        const hasFilter = options.limit !== 'all' || options.filter || options.search;
+        const ids = hasFilter ? await this._members.findFilteredIds(options) : null;
+        if (ids) {
+            logging.info(`[MembersExporter] Found ${ids.length} members matching filter criteria`);
+        }
+        const reference = await this.fetchReferenceData();
+        const membersQuery = this._knex('members').select(...MEMBER_COLUMNS);
+        if (ids) {
+            membersQuery.whereIn('id', ids);
+        }
+        logging.info('[MembersExporter] Starting streaming export of members');
+        const batchingTransform = this.createBatchingTransform();
+        const processingTransform = this.createProcessingTransform(reference);
+        (0, node_stream_1.pipeline)(membersQuery.stream(), batchingTransform, processingTransform, (err) => {
+            if (err) {
+                logging.error({ event: { name: 'members-export.stream.error' }, err }, 'Members export stream failed');
+            }
+            else {
+                logging.info('[MembersExporter] Total time taken for member export: ' + (Date.now() - start) / 1000 + 's');
+            }
+        });
+        return processingTransform;
+    }
+    // products and labels are small, stable tables, read once up front as id->name
+    // maps. Custom field definitions are read once too, which fixes the column set for
+    // the whole file: archiving a field mid-export cannot leave the header ragged. It
+    // can still leave that field's cells empty from the batch after the archive
+    // onwards, because the per-batch value read applies its own active filter.
+    async fetchReferenceData() {
+        const start = Date.now();
+        const allProducts = await this._knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => {
+            acc[product.id] = product.name;
+            return acc;
+        }, {}));
+        const allLabels = await this._knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => {
+            acc[label.id] = label.name;
+            return acc;
+        }, {}));
+        logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - start) + 'ms');
+        const activeCustomFields = await this._customFields.activeDefinitions();
+        return { allProducts, allLabels, activeCustomFields };
+    }
+    // Group the member stream into batches so the related-data reads are one query per
+    // batch rather than one per member.
+    createBatchingTransform() {
+        let currentBatch = [];
+        return new node_stream_1.Transform({
+            objectMode: true,
+            transform(member, encoding, callback) {
+                currentBatch.push(member);
+                if (currentBatch.length >= BATCH_SIZE) {
+                    this.push(currentBatch);
+                    currentBatch = [];
+                }
+                callback();
+            },
+            flush(callback) {
+                if (currentBatch.length > 0) {
+                    this.push(currentBatch);
+                }
+                callback();
+            }
+        });
+    }
+    // Each flattened row is pushed individually so a batch's rows do not pile up in memory.
+    createProcessingTransform(reference) {
+        const assembleRelatedData = (memberIds) => this.assembleRelatedData(memberIds, reference.activeCustomFields);
+        const flattenBatch = (members, related) => this.flattenBatch(members, related, reference);
+        return new node_stream_1.Transform({
+            objectMode: true,
+            highWaterMark: BATCH_SIZE,
+            async transform(batch, encoding, callback) {
+                try {
+                    const memberIds = batch.map(member => member.id);
+                    const related = await assembleRelatedData(memberIds);
+                    const rows = flattenBatch(batch, related);
+                    rows.forEach(row => this.push(row));
+                    callback();
+                }
+                catch (err) {
+                    callback(err instanceof Error ? err : new errors.InternalServerError({ message: String(err) }));
+                }
+            }
+        });
+    }
+    // One query per related table for the whole batch, not one per member. Custom field
+    // values are read only when there are columns to fill.
+    async assembleRelatedData(memberIds, activeCustomFields) {
+        const knex = this._knex;
+        const [tiers, labels, stripeCustomers, subscriptions, gifts, customFieldValuesMap] = await Promise.all([
+            knex('members_products')
+                .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
+                .whereIn('member_id', memberIds)
+                .groupBy('member_id'),
+            knex('members_labels')
+                .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
+                .whereIn('member_id', memberIds)
+                .groupBy('member_id'),
+            knex('members_stripe_customers')
+                .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
+                .whereIn('member_id', memberIds)
+                .groupBy('member_id'),
+            knex('members_newsletters')
+                .distinct('member_id')
+                .whereIn('member_id', memberIds),
+            knex('gifts')
+                .select('id', 'redeemer_member_id')
+                .where('status', 'redeemed')
+                .whereIn('redeemer_member_id', memberIds),
+            activeCustomFields.length > 0
+                ? this._customFields.valuesForMembers(memberIds)
+                : new Map()
+        ]);
+        return {
+            tiersMap: new Map(tiers.map((row) => [row.member_id, row.tiers])),
+            labelsMap: new Map(labels.map((row) => [row.member_id, row.labels])),
+            stripeCustomerMap: new Map(stripeCustomers.map((row) => [row.member_id, row.stripe_customer_id])),
+            subscribedSet: new Set(subscriptions.map((row) => row.member_id)),
+            giftIdMap: new Map(gifts.map((row) => [row.redeemer_member_id, row.id])),
+            customFieldValuesMap
+        };
+    }
+    flattenBatch(members, related, reference) {
+        const { tiersMap, labelsMap, stripeCustomerMap, subscribedSet, giftIdMap, customFieldValuesMap } = related;
+        const { allProducts, allLabels, activeCustomFields } = reference;
+        return members.map((row) => {
+            const tierConcat = tiersMap.get(row.id);
+            const tierIds = tierConcat ? tierConcat.split(',') : [];
+            const labelConcat = labelsMap.get(row.id);
+            const labelIds = labelConcat ? labelConcat.split(',') : [];
+            return {
+                ...row,
+                subscribed: subscribedSet.has(row.id),
+                comped: row.status === 'comped',
+                gift_id: giftIdMap.get(row.id) || null,
+                stripe_customer_id: stripeCustomerMap.get(row.id) || null,
+                created_at: (0, moment_1.default)(row.created_at).toISOString(),
+                tiers: tierIds.map(id => ({ name: allProducts[id] })),
+                labels: labelIds.map(id => ({ name: allLabels[id] })),
+                // Flattened here rather than in the serializer because the column set is
+                // only knowable from the database. Every member carries a cell for every
+                // active field's column, so a member with no values still contributes the
+                // full set -- the CSV header is taken from whichever row streams first.
+                custom_field_cells: csvCellsForFields(activeCustomFields, customFieldValuesMap.get(row.id) || {})
+            };
+        });
+    }
+}
+exports.default = MembersCSVExporter;