ghost 6.57.1 → 6.58.0

package/core/server/services/members/import-export/export/exporter.ts

@@ -1,325 +0,0 @@
-import moment from 'moment';
-import {Transform, pipeline, type Readable} from 'node:stream';
-import type {Knex} from 'knex';
-
-const logging = require('@tryghost/logging');
-const errors = require('@tryghost/errors');
-const {csvCellsForFields} = require('@tryghost/custom-field-types/csv');
-
-// Options accepted by the export, forwarded to the members query for filtering.
-export interface ExportOptions {
-    limit?: number | 'all';
-    filter?: string;
-    search?: string;
-    [option: string]: unknown;
-}
-
-// A custom field definition, opaque to the exporter: it only counts the set and hands
-// it to the values codec, which decides the columns. Named for intent; its shape is
-// the custom fields service's concern, not this one's.
-export type CustomFieldDefinition = Record<string, unknown>;
-
-// The custom fields collaborator: the active column set for this export, and the
-// per-member values that fill those columns. Both resolve empty when the feature is
-// off, so the exporter carries no flag of its own.
-export interface CustomFieldsService {
-    activeDefinitions(): Promise<CustomFieldDefinition[]>;
-    valuesForMembers(memberIds: string[]): Promise<Map<string, Record<string, unknown>>>;
-}
-
-// The members aggregate, narrowed to the one read the export makes: the ids matching
-// a filter, resolved without any of the related data the stream fetches itself.
-export interface MembersRepository {
-    findFilteredIds(options: ExportOptions): Promise<string[]>;
-}
-
-export interface ExporterDeps {
-    knex: Knex;
-    members: MembersRepository;
-    customFields: CustomFieldsService;
-}
-
-interface BatchRelatedData {
-    tiersMap: Map<string, string>;
-    labelsMap: Map<string, string>;
-    stripeCustomerMap: Map<string, string>;
-    subscribedSet: Set<string>;
-    giftIdMap: Map<string, string>;
-    customFieldValuesMap: Map<string, Record<string, unknown>>;
-}
-
-// The reference data read once up front and shared across every batch.
-interface ReferenceData {
-    allProducts: Record<string, string>;
-    allLabels: Record<string, string>;
-    activeCustomFields: CustomFieldDefinition[];
-}
-
-// A member row as the export query selects it -- the shape the stream emits, and the
-// contract flattenBatch is total over. Its value types are declared here (knex streams
-// rows untyped); a change to the members table that this export should carry is made by
-// changing this type, which then ripples into flattenBatch until every field is handled.
-// The end-to-end e2e export test is what catches the table drifting from this shape.
-interface MemberDbRow {
-    id: string;
-    email: string;
-    name: string | null;
-    note: string | null;
-    status: string;
-    created_at: Date | string;
-}
-
-// The columns the export reads off the members table, checked against MemberDbRow so the
-// query cannot select a column the row type does not declare.
-const MEMBER_COLUMNS = ['id', 'email', 'name', 'note', 'status', 'created_at'] satisfies Array<keyof MemberDbRow>;
-
-// The db row with its related data resolved onto it. custom_field_cells later spreads
-// into one column per active custom field.
-interface MemberExportRow extends MemberDbRow {
-    created_at: string;
-    tiers: Array<{name: string}>;
-    labels: Array<{name: string}>;
-    subscribed: boolean;
-    comped: boolean;
-    gift_id: string | null;
-    stripe_customer_id: string | null;
-    custom_field_cells: Record<string, unknown>;
-}
-
-// One row of the members export CSV: a flattened export row encoded into cells. This
-// type is the export's column contract -- the output serializer takes the columns from
-// these keys. The fixed member columns are typed; custom_field_cells spreads in whatever
-// per-site columns the database holds, so the row also carries an open string index.
-type ExportCsvRow = {
-    id: string;
-    email: string;
-    name: string | null;
-    note: string | null;
-    subscribed_to_emails: string;
-    complimentary_plan: string;
-    stripe_customer_id: string | null;
-    created_at: string;
-    deleted_at: null;
-    labels: string;
-    tiers: string;
-    gift_id: string;
-} & Record<string, unknown>;
-
-function namesToCsv(items: Array<{name: string}>): string {
-    return items.map(item => item.name).join(',');
-}
-
-// Encode a flattened export row into its CSV cells, type-checked against MemberExportRow:
-// a change to that row must be handled here or this stops compiling. deleted_at is always
-// null -- the export reads live members -- and complimentary_plan is written from comped.
-export function toExportCsvRow(row: MemberExportRow): ExportCsvRow {
-    return {
-        id: row.id,
-        email: row.email,
-        name: row.name,
-        note: row.note,
-        subscribed_to_emails: row.subscribed === true ? 'true' : 'false',
-        complimentary_plan: row.comped === true ? 'true' : '',
-        stripe_customer_id: row.stripe_customer_id,
-        created_at: row.created_at,
-        deleted_at: null,
-        labels: namesToCsv(row.labels),
-        tiers: namesToCsv(row.tiers),
-        gift_id: row.gift_id || '',
-        ...row.custom_field_cells
-    };
-}
-
-const BATCH_SIZE = 1000;
-
-// Streams the matching members as flat rows, each carrying its related data (tiers,
-// labels, subscription, gift, Stripe customer, custom fields) for the output serializer
-// to turn into CSV lines. The members aggregate and custom fields are reached only
-// through the ports above, so nothing Bookshelf- or flag-shaped leaks into the export.
-export default class MembersCSVExporter {
-    private _knex: Knex;
-    private _members: MembersRepository;
-    private _customFields: CustomFieldsService;
-
-    constructor({knex, members, customFields}: ExporterDeps) {
-        this._knex = knex;
-        this._members = members;
-        this._customFields = customFields;
-    }
-
-    async export(options: ExportOptions = {}): Promise<Readable> {
-        const start = Date.now();
-        const hasFilter = options.limit !== 'all' || options.filter || options.search;
-        const ids = hasFilter ? await this._members.findFilteredIds(options) : null;
-        if (ids) {
-            logging.info(`[MembersExporter] Found ${ids.length} members matching filter criteria`);
-        }
-
-        const reference = await this.fetchReferenceData();
-
-        const membersQuery = this._knex('members').select(...MEMBER_COLUMNS);
-        if (ids) {
-            membersQuery.whereIn('id', ids);
-        }
-
-        logging.info('[MembersExporter] Starting streaming export of members');
-        const batchingTransform = this.createBatchingTransform();
-        const processingTransform = this.createProcessingTransform(reference);
-
-        pipeline(membersQuery.stream(), batchingTransform, processingTransform, (err) => {
-            if (err) {
-                logging.error({event: {name: 'members-export.stream.error'}, err}, 'Members export stream failed');
-            } else {
-                logging.info('[MembersExporter] Total time taken for member export: ' + (Date.now() - start) / 1000 + 's');
-            }
-        });
-
-        return processingTransform;
-    }
-
-    // products and labels are small, stable tables, read once up front as id->name
-    // maps. Custom field definitions are read once too, which fixes the column set for
-    // the whole file: archiving a field mid-export cannot leave the header ragged. It
-    // can still leave that field's cells empty from the batch after the archive
-    // onwards, because the per-batch value read applies its own active filter.
-    private async fetchReferenceData(): Promise<ReferenceData> {
-        const start = Date.now();
-
-        const allProducts = await this._knex('products').select('id', 'name').then(rows => rows.reduce((acc: Record<string, string>, product: {id: string; name: string}) => {
-            acc[product.id] = product.name;
-            return acc;
-        }, {}));
-
-        const allLabels = await this._knex('labels').select('id', 'name').then(rows => rows.reduce((acc: Record<string, string>, label: {id: string; name: string}) => {
-            acc[label.id] = label.name;
-            return acc;
-        }, {}));
-
-        logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - start) + 'ms');
-
-        const activeCustomFields = await this._customFields.activeDefinitions();
-
-        return {allProducts, allLabels, activeCustomFields};
-    }
-
-    // Group the member stream into batches so the related-data reads are one query per
-    // batch rather than one per member.
-    private createBatchingTransform(): Transform {
-        let currentBatch: MemberDbRow[] = [];
-
-        return new Transform({
-            objectMode: true,
-            transform(member: MemberDbRow, encoding, callback) {
-                currentBatch.push(member);
-
-                if (currentBatch.length >= BATCH_SIZE) {
-                    this.push(currentBatch);
-                    currentBatch = [];
-                }
-
-                callback();
-            },
-            flush(callback) {
-                if (currentBatch.length > 0) {
-                    this.push(currentBatch);
-                }
-                callback();
-            }
-        });
-    }
-
-    // Each flattened row is pushed individually so a batch's rows do not pile up in memory.
-    private createProcessingTransform(reference: ReferenceData): Transform {
-        const assembleRelatedData = (memberIds: string[]) => this.assembleRelatedData(memberIds, reference.activeCustomFields);
-        const flattenBatch = (members: MemberDbRow[], related: BatchRelatedData) => this.flattenBatch(members, related, reference);
-
-        return new Transform({
-            objectMode: true,
-            highWaterMark: BATCH_SIZE,
-            async transform(batch: MemberDbRow[], encoding, callback) {
-                try {
-                    const memberIds = batch.map(member => member.id);
-                    const related = await assembleRelatedData(memberIds);
-                    const rows = flattenBatch(batch, related);
-                    rows.forEach(row => this.push(row));
-                    callback();
-                } catch (err) {
-                    callback(err instanceof Error ? err : new errors.InternalServerError({message: String(err)}));
-                }
-            }
-        });
-    }
-
-    // One query per related table for the whole batch, not one per member. Custom field
-    // values are read only when there are columns to fill.
-    private async assembleRelatedData(memberIds: string[], activeCustomFields: CustomFieldDefinition[]): Promise<BatchRelatedData> {
-        const knex = this._knex;
-
-        const [tiers, labels, stripeCustomers, subscriptions, gifts, customFieldValuesMap] = await Promise.all([
-            knex('members_products')
-                .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
-                .whereIn('member_id', memberIds)
-                .groupBy('member_id'),
-
-            knex('members_labels')
-                .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
-                .whereIn('member_id', memberIds)
-                .groupBy('member_id'),
-
-            knex('members_stripe_customers')
-                .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
-                .whereIn('member_id', memberIds)
-                .groupBy('member_id'),
-
-            knex('members_newsletters')
-                .distinct('member_id')
-                .whereIn('member_id', memberIds),
-
-            knex('gifts')
-                .select('id', 'redeemer_member_id')
-                .where('status', 'redeemed')
-                .whereIn('redeemer_member_id', memberIds),
-
-            activeCustomFields.length > 0
-                ? this._customFields.valuesForMembers(memberIds)
-                : new Map<string, Record<string, unknown>>()
-        ]);
-
-        return {
-            tiersMap: new Map(tiers.map((row: {member_id: string; tiers: string}) => [row.member_id, row.tiers])),
-            labelsMap: new Map(labels.map((row: {member_id: string; labels: string}) => [row.member_id, row.labels])),
-            stripeCustomerMap: new Map(stripeCustomers.map((row: {member_id: string; stripe_customer_id: string}) => [row.member_id, row.stripe_customer_id])),
-            subscribedSet: new Set(subscriptions.map((row: {member_id: string}) => row.member_id)),
-            giftIdMap: new Map(gifts.map((row: {id: string; redeemer_member_id: string}) => [row.redeemer_member_id, row.id])),
-            customFieldValuesMap
-        };
-    }
-
-    private flattenBatch(members: MemberDbRow[], related: BatchRelatedData, reference: ReferenceData): MemberExportRow[] {
-        const {tiersMap, labelsMap, stripeCustomerMap, subscribedSet, giftIdMap, customFieldValuesMap} = related;
-        const {allProducts, allLabels, activeCustomFields} = reference;
-
-        return members.map((row) => {
-            const tierConcat = tiersMap.get(row.id);
-            const tierIds = tierConcat ? tierConcat.split(',') : [];
-            const labelConcat = labelsMap.get(row.id);
-            const labelIds = labelConcat ? labelConcat.split(',') : [];
-
-            return {
-                ...row,
-                subscribed: subscribedSet.has(row.id),
-                comped: row.status === 'comped',
-                gift_id: giftIdMap.get(row.id) || null,
-                stripe_customer_id: stripeCustomerMap.get(row.id) || null,
-                created_at: moment(row.created_at).toISOString(),
-                tiers: tierIds.map(id => ({name: allProducts[id]})),
-                labels: labelIds.map(id => ({name: allLabels[id]})),
-                // Flattened here rather than in the serializer because the column set is
-                // only knowable from the database. Every member carries a cell for every
-                // active field's column, so a member with no values still contributes the
-                // full set -- the CSV header is taken from whichever row streams first.
-                custom_field_cells: csvCellsForFields(activeCustomFields, customFieldValuesMap.get(row.id) || {})
-            };
-        });
-    }
-}

package/core/server/services/members/import-export/import/completion-email.ts

@@ -1,146 +0,0 @@
-import {serialize} from '../csv';
-import {isCustomFieldColumn} from '@tryghost/custom-field-types/csv';
-import type {MemberImportRow, ImportErrorRow, ImportLabel, Label} from './row';
-
-const emailTemplate = require('./email-template');
-
-// The finished import as the email reads it: how many imported and which rows
-// failed. Structural, so the importer's richer result satisfies it directly.
-interface ImportSummary {
-    imported: number;
-    errors: ImportErrorRow[];
-}
-
-interface CompletionEmailInput {
-    result: ImportSummary;
-    recipient: string;
-    labelName: string;
-    importLabel: ImportLabel | null;
-    urlFor: (type: string, data: unknown, absolute: boolean) => string;
-}
-
-interface EmailPayload {
-    to: string;
-    subject: string;
-    html: string;
-    forceTextContent: boolean;
-    attachments: Array<{
-        filename: string;
-        content: string;
-        contentType: string;
-        contentDisposition: string;
-    }>;
-}
-
-// Turn an ORM validation error into copy a member manager can act on. Presentation
-// only: the API response keeps the raw messages, and just the emailed report is
-// rewritten. Takes one reason at a time, so a rewrite matching to the end of its input
-// cannot consume the reason after it.
-function humaniseReason(message: string): string {
-    return message
-        .replace('Value in [members.email] cannot be blank.', 'Missing email address')
-        .replace('Value in [members.note] exceeds maximum length of 2000 characters.', '"Note" exceeds maximum length of 2000 characters')
-        .replace('Value in [members.subscribed] must be one of true, false, 0 or 1.', 'Value in "Subscribed to emails" must be "true" or "false"')
-        .replace('Validation (isEmail) failed for email', 'Invalid email address')
-        .replace(/No such customer:[\s\S]*/, 'Could not find Stripe customer');
-}
-
-function humaniseError(row: ImportErrorRow): string {
-    return row.errors.map(humaniseReason).join('\n');
-}
-
-// One row of the fixed, member-vocabulary part of the error report, in emit order.
-// papaparse takes the fixed columns from these keys, so the report cannot drift from the
-// shaper -- a new member column must be added here or the shaper stops compiling. tiers
-// and deleted_at are export-vocabulary columns an import never fills; kept always-empty
-// so the report matches the members export CSV. Any custom_fields.* columns a submitted
-// row carried are dynamic, so they are threaded in separately by buildErrorReport.
-type ErrorReportRow = {
-    id: MemberImportRow['id'];
-    email: MemberImportRow['email'];
-    name: MemberImportRow['name'];
-    note: MemberImportRow['note'];
-    subscribed_to_emails: MemberImportRow['subscribed'];
-    complimentary_plan: MemberImportRow['complimentary_plan'];
-    stripe_customer_id: MemberImportRow['stripe_customer_id'];
-    created_at: MemberImportRow['created_at'];
-    deleted_at: undefined;
-    labels: string;
-    tiers: '';
-    gift_id: string | null;
-    error: string;
-};
-
-function stringifyLabels(labels: Array<string | Label>): string {
-    return labels.map(label => (typeof label === 'string' ? label : label.name)).join(',');
-}
-
-// The custom_fields.* cells a submitted row carried, echoed untouched so a manager can
-// fix a failed row and re-upload the values they mapped.
-function customFieldCells(row: ImportErrorRow): Record<string, unknown> {
-    return Object.fromEntries(Object.entries(row).filter(([column]) => isCustomFieldColumn(column)));
-}
-
-// Shape a failed import row into its fixed error-report cells, with the raw ORM message
-// rewritten into copy the member manager can act on. Custom field cells are merged on by
-// buildErrorReport, which owns the dynamic column set.
-function toErrorReportRow(row: ImportErrorRow): ErrorReportRow {
-    return {
-        id: row.id,
-        email: row.email,
-        name: row.name,
-        note: row.note,
-        subscribed_to_emails: row.subscribed,
-        complimentary_plan: row.complimentary_plan,
-        stripe_customer_id: row.stripe_customer_id,
-        created_at: row.created_at,
-        deleted_at: undefined,
-        labels: stringifyLabels(row.labels),
-        tiers: '',
-        gift_id: row.gift_id || null,
-        error: humaniseError(row)
-    };
-}
-
-// The error report attached to the completion email: the failed rows as CSV. It shares
-// the serialiser with the export but not the shaping -- the export writes db members,
-// this echoes submitted rows. Member columns come from the shaper's keys (so the type
-// stays the single source); the custom_fields.* columns across the rows are threaded in
-// before the last error column, and each row's custom cells merged on.
-function buildErrorReport(errors: ImportErrorRow[]): string {
-    if (errors.length === 0) {
-        return serialize([]);
-    }
-    const memberColumns = Object.keys(toErrorReportRow(errors[0])).filter(column => column !== 'error');
-    const customColumns = [...new Set(errors.flatMap(row => Object.keys(customFieldCells(row))))];
-    const columns = [...memberColumns, ...customColumns, 'error'];
-    const rows = errors.map(row => ({...toErrorReportRow(row), ...customFieldCells(row)}));
-    return serialize(rows, {columns});
-}
-
-// Compose the completion email for a finished import: the summary and its links,
-// plus the attached error report. Owns how the outcome is presented, so the
-// importer yields only the result and never touches email or CSV formatting.
-export default function buildCompletionEmail({result, recipient, labelName, importLabel, urlFor}: CompletionEmailInput): EmailPayload {
-    const siteUrl = new URL(urlFor('home', null, true));
-    const membersUrl = new URL('members', urlFor('admin', null, true));
-    if (importLabel) {
-        membersUrl.searchParams.set('label', importLabel.slug);
-    }
-
-    const html = emailTemplate({result, siteUrl, membersUrl, emailRecipient: recipient, importLabel});
-    const subject = result.imported > 0 ? 'Your member import is complete' : 'Your member import was unsuccessful';
-
-    return {
-        to: recipient,
-        subject,
-        html,
-        forceTextContent: true,
-        attachments: [{
-            filename: `${labelName} - Errors.csv`,
-            content: buildErrorReport(result.errors),
-            contentType: 'text/csv',
-            contentDisposition: 'attachment'
-        }]
-    };
-}