ghost 6.54.0 → 6.54.1

package/core/server/services/members/csv/index.ts

@@ -1,3 +0,0 @@
-export {default as parse} from './parse';
-export {default as unparse} from './unparse';
-export type {Label, MemberCsvRow} from './types';

package/core/server/services/members/csv/parse.js

@@ -1,94 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = parse;
-const node_stream_1 = require("node:stream");
-const papaparse_1 = __importDefault(require("papaparse"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const transformValue = (header, value) => {
-    if (header === 'labels') {
-        if (value && typeof value === 'string') {
-            return value.split(',').map(name => ({ name }));
-        }
-        return [];
-    }
-    if (header === 'subscribed') {
-        return value.toLowerCase() !== 'false';
-    }
-    if (header === 'complimentary_plan') {
-        return value.toLowerCase() === 'true';
-    }
-    if (value === '') {
-        return null;
-    }
-    if (value === 'undefined') {
-        return null;
-    }
-    if (value.toLowerCase() === 'false') {
-        return false;
-    }
-    if (value.toLowerCase() === 'true') {
-        return true;
-    }
-    return value;
-};
-/**
- * @param path - The path to the CSV to prepare
- * @param headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
- * @param defaultLabels - A list of labels to apply to every parsed member row
- * @returns The parsed member rows, keyed by mapped header
- */
-function parse(path, headerMapping, defaultLabels = []) {
-    return new Promise(function (resolve, reject) {
-        const csvFileStream = fs_extra_1.default.createReadStream(path);
-        const csvParserStream = papaparse_1.default.parse(papaparse_1.default.NODE_STREAM_INPUT, {
-            header: true
-        });
-        const rows = [];
-        const parsedCSVStream = (0, node_stream_1.pipeline)(csvFileStream, csvParserStream, (err) => {
-            if (err) {
-                return reject(err);
-            }
-            resolve(rows);
-        });
-        parsedCSVStream.on('data', (parsedRow) => {
-            // a throw here escapes as an uncaught exception and leaves this
-            // promise forever unsettled, so it has to become a rejection
-            try {
-                const row = {};
-                for (const [header, value] of Object.entries(parsedRow)) {
-                    // papaparse gathers the overflow from a row carrying more
-                    // fields than there are headers under __parsed_extra, as an
-                    // array rather than a cell any mapping can name
-                    if (typeof value !== 'string') {
-                        continue;
-                    }
-                    // hasOwn, not `in`: a column named after an Object.prototype
-                    // member would otherwise pass as mapped and take a function
-                    // as its mapped name
-                    if (!headerMapping || !Object.hasOwn(headerMapping, header)) {
-                        continue;
-                    }
-                    const mappedHeader = headerMapping[header];
-                    row[mappedHeader] = transformValue(mappedHeader, value);
-                }
-                // skip rows with no data
-                if (!Object.keys(row).length) {
-                    return;
-                }
-                // labels is absent when the column is unmapped, and an array otherwise
-                const parsedLabels = row.labels;
-                row.labels = [
-                    ...(Array.isArray(parsedLabels) ? parsedLabels : []),
-                    ...defaultLabels
-                ];
-                rows.push(row);
-            }
-            catch (err) {
-                reject(err);
-            }
-        });
-    });
-}

package/core/server/services/members/csv/parse.ts

@@ -1,111 +0,0 @@
-import {pipeline} from 'node:stream';
-import papaparse from 'papaparse';
-import fs from 'fs-extra';
-import type {Label, MemberCsvRow} from './types';
-
-type CsvValue = string | boolean | null | Label[];
-
-const transformValue = (header: string, value: string): CsvValue => {
-    if (header === 'labels') {
-        if (value && typeof value === 'string') {
-            return value.split(',').map(name => ({name}));
-        }
-        return [];
-    }
-
-    if (header === 'subscribed') {
-        return value.toLowerCase() !== 'false';
-    }
-
-    if (header === 'complimentary_plan') {
-        return value.toLowerCase() === 'true';
-    }
-
-    if (value === '') {
-        return null;
-    }
-
-    if (value === 'undefined') {
-        return null;
-    }
-
-    if (value.toLowerCase() === 'false') {
-        return false;
-    }
-
-    if (value.toLowerCase() === 'true') {
-        return true;
-    }
-
-    return value;
-};
-
-/**
- * @param path - The path to the CSV to prepare
- * @param headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
- * @param defaultLabels - A list of labels to apply to every parsed member row
- * @returns The parsed member rows, keyed by mapped header
- */
-export default function parse(
-    path: string,
-    headerMapping?: Record<string, string>,
-    defaultLabels: Label[] = []
-): Promise<MemberCsvRow[]> {
-    return new Promise(function (resolve, reject) {
-        const csvFileStream = fs.createReadStream(path);
-        const csvParserStream = papaparse.parse(papaparse.NODE_STREAM_INPUT, {
-            header: true
-        });
-
-        const rows: MemberCsvRow[] = [];
-        const parsedCSVStream = pipeline(csvFileStream, csvParserStream, (err) => {
-            if (err) {
-                return reject(err);
-            }
-            resolve(rows);
-        });
-
-        parsedCSVStream.on('data', (parsedRow: Record<string, unknown>) => {
-            // a throw here escapes as an uncaught exception and leaves this
-            // promise forever unsettled, so it has to become a rejection
-            try {
-                const row: MemberCsvRow = {};
-
-                for (const [header, value] of Object.entries(parsedRow)) {
-                    // papaparse gathers the overflow from a row carrying more
-                    // fields than there are headers under __parsed_extra, as an
-                    // array rather than a cell any mapping can name
-                    if (typeof value !== 'string') {
-                        continue;
-                    }
-
-                    // hasOwn, not `in`: a column named after an Object.prototype
-                    // member would otherwise pass as mapped and take a function
-                    // as its mapped name
-                    if (!headerMapping || !Object.hasOwn(headerMapping, header)) {
-                        continue;
-                    }
-
-                    const mappedHeader = headerMapping[header];
-                    row[mappedHeader] = transformValue(mappedHeader, value);
-                }
-
-                // skip rows with no data
-                if (!Object.keys(row).length) {
-                    return;
-                }
-
-                // labels is absent when the column is unmapped, and an array otherwise
-                const parsedLabels = row.labels;
-                row.labels = [
-                    ...(Array.isArray(parsedLabels) ? parsedLabels : []),
-                    ...defaultLabels
-                ];
-
-                rows.push(row);
-            } catch (err) {
-                reject(err);
-            }
-        });
-    });
-}

package/core/server/services/members/csv/types.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/core/server/services/members/csv/types.ts

@@ -1,39 +0,0 @@
-export interface Label {
-    name: string;
-}
-
-/**
- * Any column the parser does not special-case is coerced, so a cell reading
- * TRUE or FALSE arrives as a boolean and an empty one as null, whatever the
- * column means. A member genuinely named TRUE parses to `name: true`.
- */
-export type ParsedValue = string | boolean | null;
-
-/**
- * The member CSV column vocabulary as it currently stands. Several fields exist
- * on one side only: tiers and deleted_at are written on export and not accepted
- * on import, import_tier is accepted on import and never written, and the email
- * subscription state is called subscribed coming in and subscribed_to_emails
- * going out. The index signature carries custom field columns, which are not
- * known ahead of time.
- */
-export interface MemberCsvRow {
-    id?: ParsedValue;
-    email?: ParsedValue;
-    name?: ParsedValue;
-    note?: ParsedValue;
-    subscribed?: boolean;
-    subscribed_to_emails?: boolean;
-    comped?: boolean;
-    complimentary_plan?: boolean;
-    stripe_customer_id?: ParsedValue;
-    subscriptions?: Array<{customer?: {id?: string}}>;
-    created_at?: ParsedValue;
-    deleted_at?: ParsedValue;
-    labels?: string | Array<string | Label>;
-    tiers?: Array<{name: string}>;
-    import_tier?: ParsedValue;
-    gift_id?: ParsedValue;
-    error?: ParsedValue;
-    [column: string]: unknown;
-}

package/core/server/services/members/csv/unparse.js

@@ -1,69 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = unparse;
-const papaparse_1 = __importDefault(require("papaparse"));
-const DEFAULT_COLUMNS = [
-    'id',
-    'email',
-    'name',
-    'note',
-    'subscribed_to_emails',
-    'complimentary_plan',
-    'stripe_customer_id',
-    'created_at',
-    'deleted_at',
-    'labels',
-    'tiers',
-    'gift_id'
-];
-function unparse(rows, columns = DEFAULT_COLUMNS, { escapeFormulae = true } = {}) {
-    const outputColumns = columns.map((column) => {
-        if (column === 'subscribed') {
-            return 'subscribed_to_emails';
-        }
-        return column;
-    });
-    if (!outputColumns.includes('error') && rows.some(row => row.error)) {
-        outputColumns.push('error');
-    }
-    const mappedRows = rows.map((row) => {
-        let labels = '';
-        if (typeof row.labels === 'string') {
-            labels = row.labels;
-        }
-        else if (Array.isArray(row.labels)) {
-            labels = row.labels.map((l) => {
-                return typeof l === 'string' ? l : l.name;
-            }).join(',');
-        }
-        let tiers = '';
-        if (Array.isArray(row.tiers)) {
-            tiers = row.tiers.map((tier) => {
-                return tier.name;
-            }).join(',');
-        }
-        return {
-            id: row.id,
-            email: row.email,
-            name: row.name,
-            note: row.note,
-            subscribed_to_emails: 'subscribed' in row ? row.subscribed : row.subscribed_to_emails,
-            complimentary_plan: row.comped || row.complimentary_plan,
-            stripe_customer_id: row.subscriptions?.[0]?.customer?.id || row.stripe_customer_id,
-            created_at: row.created_at,
-            deleted_at: row.deleted_at,
-            labels,
-            tiers,
-            import_tier: row.import_tier || null,
-            gift_id: row.gift_id || null,
-            error: row.error || null
-        };
-    });
-    return papaparse_1.default.unparse(mappedRows, {
-        escapeFormulae,
-        columns: outputColumns
-    });
-}

package/core/server/services/members/csv/unparse.ts

@@ -1,79 +0,0 @@
-import papaparse from 'papaparse';
-import type {MemberCsvRow} from './types';
-
-const DEFAULT_COLUMNS = [
-    'id',
-    'email',
-    'name',
-    'note',
-    'subscribed_to_emails',
-    'complimentary_plan',
-    'stripe_customer_id',
-    'created_at',
-    'deleted_at',
-    'labels',
-    'tiers',
-    'gift_id'
-];
-
-interface UnparseOptions {
-    // Prefixes a leading =, +, -, @ or tab with an apostrophe so a spreadsheet does
-    // not read the value as a formula. On by default: it belongs on any CSV a person
-    // opens. The importer's own intermediate file is not one, and turns it off, so a
-    // member named "-5" is not read back as "'-5".
-    escapeFormulae?: boolean;
-}
-
-export default function unparse(rows: MemberCsvRow[], columns: string[] = DEFAULT_COLUMNS, {escapeFormulae = true}: UnparseOptions = {}): string {
-    const outputColumns = columns.map((column) => {
-        if (column === 'subscribed') {
-            return 'subscribed_to_emails';
-        }
-        return column;
-    });
-
-    if (!outputColumns.includes('error') && rows.some(row => row.error)) {
-        outputColumns.push('error');
-    }
-
-    const mappedRows = rows.map((row) => {
-        let labels = '';
-        if (typeof row.labels === 'string') {
-            labels = row.labels;
-        } else if (Array.isArray(row.labels)) {
-            labels = row.labels.map((l) => {
-                return typeof l === 'string' ? l : l.name;
-            }).join(',');
-        }
-
-        let tiers = '';
-
-        if (Array.isArray(row.tiers)) {
-            tiers = row.tiers.map((tier) => {
-                return tier.name;
-            }).join(',');
-        }
-
-        return {
-            id: row.id,
-            email: row.email,
-            name: row.name,
-            note: row.note,
-            subscribed_to_emails: 'subscribed' in row ? row.subscribed : row.subscribed_to_emails,
-            complimentary_plan: row.comped || row.complimentary_plan,
-            stripe_customer_id: row.subscriptions?.[0]?.customer?.id || row.stripe_customer_id,
-            created_at: row.created_at,
-            deleted_at: row.deleted_at,
-            labels,
-            tiers,
-            import_tier: row.import_tier || null,
-            gift_id: row.gift_id || null,
-            error: row.error || null
-        };
-    });
-
-    return papaparse.unparse(mappedRows, {
-        escapeFormulae,
-        columns: outputColumns
-    });
-}

package/core/server/services/members/exporter/query.js

@@ -1,233 +0,0 @@
-const models = require('../../../models');
-const {knex} = require('../../../data/db');
-const moment = require('moment');
-const logging = require('@tryghost/logging');
-const {Transform} = require('stream');
-const labs = require('../../../../shared/labs');
-const {csvCellsForFields} = require('@tryghost/custom-field-types/csv');
-
-/**
- * Process a batch of members with all their related data
- * @param {Array} members - Array of member records
- * @param {Object} tiersMap - Map of member_id to tiers
- * @param {Object} labelsMap - Map of member_id to labels
- * @param {Object} stripeCustomerMap - Map of member_id to stripe customer IDs
- * @param {Set} subscribedSet - Set of subscribed member IDs
- * @param {Map} giftIdMap - Map of member_id to gift id (for gift members only)
- * @param {Object} allProducts - Map of product IDs to product names
- * @param {Object} allLabels - Map of label IDs to label names
- * @param {Array} activeCustomFields - Active custom field definitions
- * @param {Map} customFieldValuesMap - Map of member_id to their custom field values
- * @returns {Array} Processed member records
- */
-function processMembersData(members, tiersMap, labelsMap, stripeCustomerMap, subscribedSet, giftIdMap, allProducts, allLabels, activeCustomFields, customFieldValuesMap) {
-    for (const row of members) {
-        const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : [];
-        const tierDetails = tierIds.map((id) => {
-            return {
-                name: allProducts[id]
-            };
-        });
-        row.tiers = tierDetails;
-
-        const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : [];
-        const labelDetails = labelIds.map((id) => {
-            return {
-                name: allLabels[id]
-            };
-        });
-        row.labels = labelDetails;
-
-        row.subscribed = subscribedSet.has(row.id);
-        row.comped = row.status === 'comped';
-        row.complimentary_plan = row.status === 'complimentary';
-        row.gift_id = giftIdMap.get(row.id) || null;
-        row.stripe_customer_id = stripeCustomerMap.get(row.id) || null;
-        row.created_at = moment(row.created_at).toISOString();
-
-        // Flattened here rather than in the serializer because the column set is
-        // only knowable from the database. Every member carries a cell for every
-        // active field's column, so a member with no values still contributes the
-        // full set — the CSV header is taken from whichever row streams first.
-        row.custom_field_cells = csvCellsForFields(activeCustomFields, customFieldValuesMap.get(row.id) || {});
-    }
-    return members;
-}
-
-/**
- * Creates a streaming implementation of the members export
- * @param {Object} options - Export options
- * @returns {Promise<Object>} A readable stream of processed members
- */
-async function createExportStream(options, {definitions, values}) {
-    const hasFilter = options.limit !== 'all' || options.filter || options.search;
-    const start = Date.now();
-    
-    let ids = null;
-    if (hasFilter) {
-        // do a very minimal query, only to fetch the ids of the filtered values
-        const filterOptions = {...options};
-        filterOptions.withRelated = [];
-        filterOptions.columns = ['id'];
-        
-        // Important: We need to get ALL ids, not just the first page
-        filterOptions.limit = 'all';
-
-        const page = await models.Member.findPage(filterOptions);
-        ids = page.data.map(d => d.id);
-        logging.info(`[MembersExporter] Found ${ids.length} members matching filter criteria`);
-    }
-
-    const startFetchingProducts = Date.now();
-
-    // Get all products and labels upfront as these are small tables
-    const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => {
-        acc[product.id] = product.name;
-        return acc; 
-    }, {}));
-
-    const allLabels = await knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => {
-        acc[label.id] = label.name;
-        return acc;
-    }, {}));
-
-    logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms');
-
-    // Definitions are a small, stable table, so they're read once rather than per
-    // batch. Reading once also fixes the column set for the whole file: archiving a
-    // field mid-export can't leave the header ragged. It can still leave that
-    // field's cells empty from the batch after the archive onwards, because the
-    // per-batch value read applies its own active filter.
-    const activeCustomFields = labs.isSet('membersCustomFields')
-        ? await definitions.browse()
-        : [];
-
-    // Create a transform stream that will process the members as they come in
-    const processMembersTransform = new Transform({
-        objectMode: true,
-        highWaterMark: 1000, // Process 1000 members at a time
-        async transform(batch, encoding, callback) {
-            try {
-                const memberIds = batch.map(member => member.id);
-                
-                // Fetch related data for this batch
-                const [tiers, labels, stripeCustomers, subscriptions, gifts, customFieldValuesMap] = await Promise.all([
-                    knex('members_products')
-                        .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
-                        .whereIn('member_id', memberIds)
-                        .groupBy('member_id'),
-                    
-                    knex('members_labels')
-                        .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
-                        .whereIn('member_id', memberIds)
-                        .groupBy('member_id'),
-                    
-                    knex('members_stripe_customers')
-                        .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
-                        .whereIn('member_id', memberIds)
-                        .groupBy('member_id'),
-                    
-                    knex('members_newsletters')
-                        .distinct('member_id')
-                        .whereIn('member_id', memberIds),
-
-                    knex('gifts')
-                        .select('id', 'redeemer_member_id')
-                        .where('status', 'redeemed')
-                        .whereIn('redeemer_member_id', memberIds),
-
-                    activeCustomFields.length > 0
-                        ? values.getValuesForMembers(memberIds)
-                        : new Map()
-                ]);
-
-                // Create maps for quick lookups
-                const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers]));
-                const labelsMap = new Map(labels.map(row => [row.member_id, row.labels]));
-                const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id]));
-                const subscribedSet = new Set(subscriptions.map(row => row.member_id));
-                const giftIdMap = new Map(gifts.map(row => [row.redeemer_member_id, row.id]));
-
-                // Process the batch
-                const processedMembers = processMembersData(
-                    batch, 
-                    tiersMap, 
-                    labelsMap, 
-                    stripeCustomerMap, 
-                    subscribedSet, 
-                    giftIdMap,
-                    allProducts,
-                    allLabels,
-                    activeCustomFields,
-                    customFieldValuesMap
-                );
-
-                // Push each member individually to avoid large arrays in memory
-                processedMembers.forEach((member) => {
-                    this.push(member);
-                });
-                
-                callback();
-            } catch (err) {
-                callback(err);
-            }
-        }
-    });
-
-    // Create a batching stream to group members
-    const batchSize = 1000; // Process in batches of 1000
-    let currentBatch = [];
-    
-    const batchingTransform = new Transform({
-        objectMode: true,
-        transform(member, encoding, callback) {
-            currentBatch.push(member);
-            
-            if (currentBatch.length >= batchSize) {
-                this.push(currentBatch);
-                currentBatch = [];
-            }
-            
-            callback();
-        },
-        flush(callback) {
-            if (currentBatch.length > 0) {
-                this.push(currentBatch);
-            }
-            callback();
-        }
-    });
-
-    // Create a query stream for members
-    const membersQuery = knex('members')
-        .select('id', 'email', 'name', 'note', 'status', 'created_at');
-    
-    if (hasFilter) {
-        membersQuery.whereIn('id', ids);
-    }
-
-    logging.info('[MembersExporter] Starting streaming export of members');
-    
-    // Chain the streams together
-    const membersStream = membersQuery.stream()
-        .pipe(batchingTransform)
-        .pipe(processMembersTransform);
-
-    // Log when export is complete
-    membersStream.on('end', () => {
-        logging.info('[MembersExporter] Total time taken for member export: ' + (Date.now() - start) / 1000 + 's');
-    });
-
-    return membersStream;
-}
-
-/**
- * Export members data
- * @param {Object} options - Export options
- * @returns {Promise<Array|Object>} Array of members or stream of members
- */
-module.exports = function createExporter({definitions, values}) {
-    return async function (options = {}) {
-        return createExportStream(options, {definitions, values});
-    };
-};

package/core/server/services/members/importer/index.js

@@ -1,30 +0,0 @@
-const MembersCSVImporter = require('./members-csv-importer');
-const MembersCSVImporterStripeUtils = require('./members-csv-importer-stripe-utils');
-
-/**
- * @typedef {import('./members-csv-importer').MembersCSVImporterOptions} MembersCSVImporterOptions
- */
-
-/**
- * @typedef {Object} MakeImporterDeps
- * @property {Object} stripeAPIService - Instance of StripeAPIService
- * @property {Object} productRepository - Instance of ProductRepository
- */
-
-/**
- * Make an instance of MembersCSVImporter
- *
- * @param {MakeImporterDeps & MembersCSVImporterOptions} deps
- * @returns {MembersCSVImporter}
- */
-module.exports = function makeImporter(deps) {
-    const stripeUtils = new MembersCSVImporterStripeUtils({
-        stripeAPIService: deps.stripeAPIService,
-        productRepository: deps.productRepository
-    });
-
-    return new MembersCSVImporter({
-        ...deps,
-        stripeUtils
-    });
-};