ghost 6.53.0 → 6.54.0

package/core/server/services/members/csv/parse.js

@@ -0,0 +1,94 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = parse;
+const node_stream_1 = require("node:stream");
+const papaparse_1 = __importDefault(require("papaparse"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const transformValue = (header, value) => {
+    if (header === 'labels') {
+        if (value && typeof value === 'string') {
+            return value.split(',').map(name => ({ name }));
+        }
+        return [];
+    }
+    if (header === 'subscribed') {
+        return value.toLowerCase() !== 'false';
+    }
+    if (header === 'complimentary_plan') {
+        return value.toLowerCase() === 'true';
+    }
+    if (value === '') {
+        return null;
+    }
+    if (value === 'undefined') {
+        return null;
+    }
+    if (value.toLowerCase() === 'false') {
+        return false;
+    }
+    if (value.toLowerCase() === 'true') {
+        return true;
+    }
+    return value;
+};
+/**
+ * @param path - The path to the CSV to prepare
+ * @param headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
+ * @param defaultLabels - A list of labels to apply to every parsed member row
+ * @returns The parsed member rows, keyed by mapped header
+ */
+function parse(path, headerMapping, defaultLabels = []) {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs_extra_1.default.createReadStream(path);
+        const csvParserStream = papaparse_1.default.parse(papaparse_1.default.NODE_STREAM_INPUT, {
+            header: true
+        });
+        const rows = [];
+        const parsedCSVStream = (0, node_stream_1.pipeline)(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+        parsedCSVStream.on('data', (parsedRow) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row = {};
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+                    // hasOwn, not `in`: a column named after an Object.prototype
+                    // member would otherwise pass as mapped and take a function
+                    // as its mapped name
+                    if (!headerMapping || !Object.hasOwn(headerMapping, header)) {
+                        continue;
+                    }
+                    const mappedHeader = headerMapping[header];
+                    row[mappedHeader] = transformValue(mappedHeader, value);
+                }
+                // skip rows with no data
+                if (!Object.keys(row).length) {
+                    return;
+                }
+                // labels is absent when the column is unmapped, and an array otherwise
+                const parsedLabels = row.labels;
+                row.labels = [
+                    ...(Array.isArray(parsedLabels) ? parsedLabels : []),
+                    ...defaultLabels
+                ];
+                rows.push(row);
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/csv/parse.ts

@@ -0,0 +1,111 @@
+import {pipeline} from 'node:stream';
+import papaparse from 'papaparse';
+import fs from 'fs-extra';
+import type {Label, MemberCsvRow} from './types';
+
+type CsvValue = string | boolean | null | Label[];
+
+const transformValue = (header: string, value: string): CsvValue => {
+    if (header === 'labels') {
+        if (value && typeof value === 'string') {
+            return value.split(',').map(name => ({name}));
+        }
+        return [];
+    }
+
+    if (header === 'subscribed') {
+        return value.toLowerCase() !== 'false';
+    }
+
+    if (header === 'complimentary_plan') {
+        return value.toLowerCase() === 'true';
+    }
+
+    if (value === '') {
+        return null;
+    }
+
+    if (value === 'undefined') {
+        return null;
+    }
+
+    if (value.toLowerCase() === 'false') {
+        return false;
+    }
+
+    if (value.toLowerCase() === 'true') {
+        return true;
+    }
+
+    return value;
+};
+
+/**
+ * @param path - The path to the CSV to prepare
+ * @param headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
+ * @param defaultLabels - A list of labels to apply to every parsed member row
+ * @returns The parsed member rows, keyed by mapped header
+ */
+export default function parse(
+    path: string,
+    headerMapping?: Record<string, string>,
+    defaultLabels: Label[] = []
+): Promise<MemberCsvRow[]> {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs.createReadStream(path);
+        const csvParserStream = papaparse.parse(papaparse.NODE_STREAM_INPUT, {
+            header: true
+        });
+
+        const rows: MemberCsvRow[] = [];
+        const parsedCSVStream = pipeline(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+
+        parsedCSVStream.on('data', (parsedRow: Record<string, unknown>) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row: MemberCsvRow = {};
+
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+
+                    // hasOwn, not `in`: a column named after an Object.prototype
+                    // member would otherwise pass as mapped and take a function
+                    // as its mapped name
+                    if (!headerMapping || !Object.hasOwn(headerMapping, header)) {
+                        continue;
+                    }
+
+                    const mappedHeader = headerMapping[header];
+                    row[mappedHeader] = transformValue(mappedHeader, value);
+                }
+
+                // skip rows with no data
+                if (!Object.keys(row).length) {
+                    return;
+                }
+
+                // labels is absent when the column is unmapped, and an array otherwise
+                const parsedLabels = row.labels;
+                row.labels = [
+                    ...(Array.isArray(parsedLabels) ? parsedLabels : []),
+                    ...defaultLabels
+                ];
+
+                rows.push(row);
+            } catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/csv/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/core/server/services/members/csv/types.ts

@@ -0,0 +1,39 @@
+export interface Label {
+    name: string;
+}
+
+/**
+ * Any column the parser does not special-case is coerced, so a cell reading
+ * TRUE or FALSE arrives as a boolean and an empty one as null, whatever the
+ * column means. A member genuinely named TRUE parses to `name: true`.
+ */
+export type ParsedValue = string | boolean | null;
+
+/**
+ * The member CSV column vocabulary as it currently stands. Several fields exist
+ * on one side only: tiers and deleted_at are written on export and not accepted
+ * on import, import_tier is accepted on import and never written, and the email
+ * subscription state is called subscribed coming in and subscribed_to_emails
+ * going out. The index signature carries custom field columns, which are not
+ * known ahead of time.
+ */
+export interface MemberCsvRow {
+    id?: ParsedValue;
+    email?: ParsedValue;
+    name?: ParsedValue;
+    note?: ParsedValue;
+    subscribed?: boolean;
+    subscribed_to_emails?: boolean;
+    comped?: boolean;
+    complimentary_plan?: boolean;
+    stripe_customer_id?: ParsedValue;
+    subscriptions?: Array<{customer?: {id?: string}}>;
+    created_at?: ParsedValue;
+    deleted_at?: ParsedValue;
+    labels?: string | Array<string | Label>;
+    tiers?: Array<{name: string}>;
+    import_tier?: ParsedValue;
+    gift_id?: ParsedValue;
+    error?: ParsedValue;
+    [column: string]: unknown;
+}

package/core/server/services/members/csv/unparse.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = unparse;
+const papaparse_1 = __importDefault(require("papaparse"));
+const DEFAULT_COLUMNS = [
+    'id',
+    'email',
+    'name',
+    'note',
+    'subscribed_to_emails',
+    'complimentary_plan',
+    'stripe_customer_id',
+    'created_at',
+    'deleted_at',
+    'labels',
+    'tiers',
+    'gift_id'
+];
+function unparse(rows, columns = DEFAULT_COLUMNS, { escapeFormulae = true } = {}) {
+    const outputColumns = columns.map((column) => {
+        if (column === 'subscribed') {
+            return 'subscribed_to_emails';
+        }
+        return column;
+    });
+    if (!outputColumns.includes('error') && rows.some(row => row.error)) {
+        outputColumns.push('error');
+    }
+    const mappedRows = rows.map((row) => {
+        let labels = '';
+        if (typeof row.labels === 'string') {
+            labels = row.labels;
+        }
+        else if (Array.isArray(row.labels)) {
+            labels = row.labels.map((l) => {
+                return typeof l === 'string' ? l : l.name;
+            }).join(',');
+        }
+        let tiers = '';
+        if (Array.isArray(row.tiers)) {
+            tiers = row.tiers.map((tier) => {
+                return tier.name;
+            }).join(',');
+        }
+        return {
+            id: row.id,
+            email: row.email,
+            name: row.name,
+            note: row.note,
+            subscribed_to_emails: 'subscribed' in row ? row.subscribed : row.subscribed_to_emails,
+            complimentary_plan: row.comped || row.complimentary_plan,
+            stripe_customer_id: row.subscriptions?.[0]?.customer?.id || row.stripe_customer_id,
+            created_at: row.created_at,
+            deleted_at: row.deleted_at,
+            labels,
+            tiers,
+            import_tier: row.import_tier || null,
+            gift_id: row.gift_id || null,
+            error: row.error || null
+        };
+    });
+    return papaparse_1.default.unparse(mappedRows, {
+        escapeFormulae,
+        columns: outputColumns
+    });
+}

package/core/server/services/members/csv/unparse.ts

@@ -0,0 +1,79 @@
+import papaparse from 'papaparse';
+import type {MemberCsvRow} from './types';
+
+const DEFAULT_COLUMNS = [
+    'id',
+    'email',
+    'name',
+    'note',
+    'subscribed_to_emails',
+    'complimentary_plan',
+    'stripe_customer_id',
+    'created_at',
+    'deleted_at',
+    'labels',
+    'tiers',
+    'gift_id'
+];
+
+interface UnparseOptions {
+    // Prefixes a leading =, +, -, @ or tab with an apostrophe so a spreadsheet does
+    // not read the value as a formula. On by default: it belongs on any CSV a person
+    // opens. The importer's own intermediate file is not one, and turns it off, so a
+    // member named "-5" is not read back as "'-5".
+    escapeFormulae?: boolean;
+}
+
+export default function unparse(rows: MemberCsvRow[], columns: string[] = DEFAULT_COLUMNS, {escapeFormulae = true}: UnparseOptions = {}): string {
+    const outputColumns = columns.map((column) => {
+        if (column === 'subscribed') {
+            return 'subscribed_to_emails';
+        }
+        return column;
+    });
+
+    if (!outputColumns.includes('error') && rows.some(row => row.error)) {
+        outputColumns.push('error');
+    }
+
+    const mappedRows = rows.map((row) => {
+        let labels = '';
+        if (typeof row.labels === 'string') {
+            labels = row.labels;
+        } else if (Array.isArray(row.labels)) {
+            labels = row.labels.map((l) => {
+                return typeof l === 'string' ? l : l.name;
+            }).join(',');
+        }
+
+        let tiers = '';
+
+        if (Array.isArray(row.tiers)) {
+            tiers = row.tiers.map((tier) => {
+                return tier.name;
+            }).join(',');
+        }
+
+        return {
+            id: row.id,
+            email: row.email,
+            name: row.name,
+            note: row.note,
+            subscribed_to_emails: 'subscribed' in row ? row.subscribed : row.subscribed_to_emails,
+            complimentary_plan: row.comped || row.complimentary_plan,
+            stripe_customer_id: row.subscriptions?.[0]?.customer?.id || row.stripe_customer_id,
+            created_at: row.created_at,
+            deleted_at: row.deleted_at,
+            labels,
+            tiers,
+            import_tier: row.import_tier || null,
+            gift_id: row.gift_id || null,
+            error: row.error || null
+        };
+    });
+
+    return papaparse.unparse(mappedRows, {
+        escapeFormulae,
+        columns: outputColumns
+    });
+}

package/core/server/services/members/exporter/query.js

@@ -3,6 +3,8 @@ const {knex} = require('../../../data/db');
 const moment = require('moment');
 const logging = require('@tryghost/logging');
 const {Transform} = require('stream');
+const labs = require('../../../../shared/labs');
+const {csvCellsForFields} = require('@tryghost/custom-field-types/csv');
 
 /**
  * Process a batch of members with all their related data
@@ -14,9 +16,11 @@ const {Transform} = require('stream');
  * @param {Map} giftIdMap - Map of member_id to gift id (for gift members only)
  * @param {Object} allProducts - Map of product IDs to product names
  * @param {Object} allLabels - Map of label IDs to label names
+ * @param {Array} activeCustomFields - Active custom field definitions
+ * @param {Map} customFieldValuesMap - Map of member_id to their custom field values
  * @returns {Array} Processed member records
  */
-function processMembersData(members, tiersMap, labelsMap, stripeCustomerMap, subscribedSet, giftIdMap, allProducts, allLabels) {
+function processMembersData(members, tiersMap, labelsMap, stripeCustomerMap, subscribedSet, giftIdMap, allProducts, allLabels, activeCustomFields, customFieldValuesMap) {
     for (const row of members) {
         const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : [];
         const tierDetails = tierIds.map((id) => {
@@ -40,6 +44,12 @@ function processMembersData(members, tiersMap, labelsMap, stripeCustomerMap, sub
         row.gift_id = giftIdMap.get(row.id) || null;
         row.stripe_customer_id = stripeCustomerMap.get(row.id) || null;
         row.created_at = moment(row.created_at).toISOString();
+
+        // Flattened here rather than in the serializer because the column set is
+        // only knowable from the database. Every member carries a cell for every
+        // active field's column, so a member with no values still contributes the
+        // full set — the CSV header is taken from whichever row streams first.
+        row.custom_field_cells = csvCellsForFields(activeCustomFields, customFieldValuesMap.get(row.id) || {});
     }
     return members;
 }
@@ -49,7 +59,7 @@ function processMembersData(members, tiersMap, labelsMap, stripeCustomerMap, sub
  * @param {Object} options - Export options
  * @returns {Promise<Object>} A readable stream of processed members
  */
-async function createExportStream(options) {
+async function createExportStream(options, {definitions, values}) {
     const hasFilter = options.limit !== 'all' || options.filter || options.search;
     const start = Date.now();
     
@@ -83,6 +93,15 @@ async function createExportStream(options) {
 
     logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms');
 
+    // Definitions are a small, stable table, so they're read once rather than per
+    // batch. Reading once also fixes the column set for the whole file: archiving a
+    // field mid-export can't leave the header ragged. It can still leave that
+    // field's cells empty from the batch after the archive onwards, because the
+    // per-batch value read applies its own active filter.
+    const activeCustomFields = labs.isSet('membersCustomFields')
+        ? await definitions.browse()
+        : [];
+
     // Create a transform stream that will process the members as they come in
     const processMembersTransform = new Transform({
         objectMode: true,
@@ -92,7 +111,7 @@ async function createExportStream(options) {
                 const memberIds = batch.map(member => member.id);
                 
                 // Fetch related data for this batch
-                const [tiers, labels, stripeCustomers, subscriptions, gifts] = await Promise.all([
+                const [tiers, labels, stripeCustomers, subscriptions, gifts, customFieldValuesMap] = await Promise.all([
                     knex('members_products')
                         .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
                         .whereIn('member_id', memberIds)
@@ -115,7 +134,11 @@ async function createExportStream(options) {
                     knex('gifts')
                         .select('id', 'redeemer_member_id')
                         .where('status', 'redeemed')
-                        .whereIn('redeemer_member_id', memberIds)
+                        .whereIn('redeemer_member_id', memberIds),
+
+                    activeCustomFields.length > 0
+                        ? values.getValuesForMembers(memberIds)
+                        : new Map()
                 ]);
 
                 // Create maps for quick lookups
@@ -133,8 +156,10 @@ async function createExportStream(options) {
                     stripeCustomerMap, 
                     subscribedSet, 
                     giftIdMap,
-                    allProducts, 
-                    allLabels
+                    allProducts,
+                    allLabels,
+                    activeCustomFields,
+                    customFieldValuesMap
                 );
 
                 // Push each member individually to avoid large arrays in memory
@@ -201,6 +226,8 @@ async function createExportStream(options) {
  * @param {Object} options - Export options
  * @returns {Promise<Array|Object>} Array of members or stream of members
  */
-module.exports = async function (options = {}) {
-    return createExportStream(options);
+module.exports = function createExporter({definitions, values}) {
+    return async function (options = {}) {
+        return createExportStream(options, {definitions, values});
+    };
 };

package/core/server/services/members/importer/config.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveInlineThreshold = resolveInlineThreshold;
+const zod_1 = require("zod");
+/**
+ * Resolving Ghost's config into the values the importer runs on.
+ *
+ * Config arrives untyped: defaults.json supplies the shipped value, and any
+ * higher layer — an operator's config file, an environment variable, argv — can
+ * replace it with anything at all. Turning that into something usable is a
+ * config concern, so it happens here, at the boundary. The importer is handed an
+ * answer and never learns where it came from.
+ */
+// The row count at or below which an import is performed while the request is
+// still open. Zero is allowed and means every import is deferred. A numeric
+// string is accepted because a value supplied through an environment variable
+// can arrive as one.
+const Threshold = zod_1.z
+    .union([zod_1.z.number(), zod_1.z.string().trim().min(1)])
+    .transform(Number)
+    .pipe(zod_1.z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER));
+// defaults.json is the single source of the shipped threshold: it is the lowest
+// config layer and the file an operator actually reads and edits, so it is read
+// rather than restated. Parsed eagerly because a missing or malformed value
+// there is our bug rather than an operator's, and should fail loudly at boot
+// instead of quietly becoming the fallback for everything below.
+const shippedThreshold = Threshold.parse(require('../../../../shared/config/defaults.json').members.importer.inlineThreshold);
+// A higher layer can override the shipped value with anything. Something
+// unreadable falls back to the shipped threshold rather than throwing, because
+// an operator can correct this live and a bad value must not take member import
+// down with it.
+const InlineThreshold = Threshold.catch(shippedThreshold);
+function resolveInlineThreshold(configured) {
+    return InlineThreshold.parse(configured);
+}

package/core/server/services/members/importer/config.ts

@@ -0,0 +1,39 @@
+import {z} from 'zod';
+
+/**
+ * Resolving Ghost's config into the values the importer runs on.
+ *
+ * Config arrives untyped: defaults.json supplies the shipped value, and any
+ * higher layer — an operator's config file, an environment variable, argv — can
+ * replace it with anything at all. Turning that into something usable is a
+ * config concern, so it happens here, at the boundary. The importer is handed an
+ * answer and never learns where it came from.
+ */
+
+// The row count at or below which an import is performed while the request is
+// still open. Zero is allowed and means every import is deferred. A numeric
+// string is accepted because a value supplied through an environment variable
+// can arrive as one.
+const Threshold = z
+    .union([z.number(), z.string().trim().min(1)])
+    .transform(Number)
+    .pipe(z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER));
+
+// defaults.json is the single source of the shipped threshold: it is the lowest
+// config layer and the file an operator actually reads and edits, so it is read
+// rather than restated. Parsed eagerly because a missing or malformed value
+// there is our bug rather than an operator's, and should fail loudly at boot
+// instead of quietly becoming the fallback for everything below.
+const shippedThreshold: number = Threshold.parse(
+    require('../../../../shared/config/defaults.json').members.importer.inlineThreshold
+);
+
+// A higher layer can override the shipped value with anything. Something
+// unreadable falls back to the shipped threshold rather than throwing, because
+// an operator can correct this live and a bad value must not take member import
+// down with it.
+const InlineThreshold = Threshold.catch(shippedThreshold);
+
+export function resolveInlineThreshold(configured: unknown): number {
+    return InlineThreshold.parse(configured);
+}