ghost 6.54.0 → 6.55.0

package/core/server/services/members/import-export/import/completion-email.ts

@@ -0,0 +1,141 @@
+import {serialize} from '../csv';
+import {isCustomFieldColumn} from '@tryghost/custom-field-types/csv';
+import type {MemberImportRow, ImportErrorRow, ImportLabel, Label} from './row';
+
+const emailTemplate = require('./email-template');
+
+// The finished import as the email reads it: how many imported and which rows
+// failed. Structural, so the importer's richer result satisfies it directly.
+interface ImportSummary {
+    imported: number;
+    errors: ImportErrorRow[];
+}
+
+interface CompletionEmailInput {
+    result: ImportSummary;
+    recipient: string;
+    labelName: string;
+    importLabel: ImportLabel | null;
+    urlFor: (type: string, data: unknown, absolute: boolean) => string;
+}
+
+interface EmailPayload {
+    to: string;
+    subject: string;
+    html: string;
+    forceTextContent: boolean;
+    attachments: Array<{
+        filename: string;
+        content: string;
+        contentType: string;
+        contentDisposition: string;
+    }>;
+}
+
+// Turn an ORM validation error into copy a member manager can act on. Presentation
+// only: the API response keeps the raw messages, and just the emailed report is
+// rewritten.
+function humaniseError(message: string): string {
+    return message
+        .replace('Value in [members.email] cannot be blank.', 'Missing email address')
+        .replace('Value in [members.note] exceeds maximum length of 2000 characters.', '"Note" exceeds maximum length of 2000 characters')
+        .replace('Value in [members.subscribed] must be one of true, false, 0 or 1.', 'Value in "Subscribed to emails" must be "true" or "false"')
+        .replace('Validation (isEmail) failed for email', 'Invalid email address')
+        .replace(/No such customer:[^,]*/, 'Could not find Stripe customer');
+}
+
+// One row of the fixed, member-vocabulary part of the error report, in emit order.
+// papaparse takes the fixed columns from these keys, so the report cannot drift from the
+// shaper -- a new member column must be added here or the shaper stops compiling. tiers
+// and deleted_at are export-vocabulary columns an import never fills; kept always-empty
+// so the report matches the members export CSV. Any custom_fields.* columns a submitted
+// row carried are dynamic, so they are threaded in separately by buildErrorReport.
+type ErrorReportRow = {
+    id: MemberImportRow['id'];
+    email: MemberImportRow['email'];
+    name: MemberImportRow['name'];
+    note: MemberImportRow['note'];
+    subscribed_to_emails: MemberImportRow['subscribed'];
+    complimentary_plan: MemberImportRow['complimentary_plan'];
+    stripe_customer_id: MemberImportRow['stripe_customer_id'];
+    created_at: MemberImportRow['created_at'];
+    deleted_at: undefined;
+    labels: string;
+    tiers: '';
+    gift_id: string | null;
+    error: string;
+};
+
+function stringifyLabels(labels: Array<string | Label>): string {
+    return labels.map(label => (typeof label === 'string' ? label : label.name)).join(',');
+}
+
+// The custom_fields.* cells a submitted row carried, echoed untouched so a manager can
+// fix a failed row and re-upload the values they mapped.
+function customFieldCells(row: ImportErrorRow): Record<string, unknown> {
+    return Object.fromEntries(Object.entries(row).filter(([column]) => isCustomFieldColumn(column)));
+}
+
+// Shape a failed import row into its fixed error-report cells, with the raw ORM message
+// rewritten into copy the member manager can act on. Custom field cells are merged on by
+// buildErrorReport, which owns the dynamic column set.
+function toErrorReportRow(row: ImportErrorRow): ErrorReportRow {
+    return {
+        id: row.id,
+        email: row.email,
+        name: row.name,
+        note: row.note,
+        subscribed_to_emails: row.subscribed,
+        complimentary_plan: row.complimentary_plan,
+        stripe_customer_id: row.stripe_customer_id,
+        created_at: row.created_at,
+        deleted_at: undefined,
+        labels: stringifyLabels(row.labels),
+        tiers: '',
+        gift_id: row.gift_id || null,
+        error: humaniseError(row.error)
+    };
+}
+
+// The error report attached to the completion email: the failed rows as CSV. It shares
+// the serialiser with the export but not the shaping -- the export writes db members,
+// this echoes submitted rows. Member columns come from the shaper's keys (so the type
+// stays the single source); the custom_fields.* columns across the rows are threaded in
+// before the last error column, and each row's custom cells merged on.
+function buildErrorReport(errors: ImportErrorRow[]): string {
+    if (errors.length === 0) {
+        return serialize([]);
+    }
+    const memberColumns = Object.keys(toErrorReportRow(errors[0])).filter(column => column !== 'error');
+    const customColumns = [...new Set(errors.flatMap(row => Object.keys(customFieldCells(row))))];
+    const columns = [...memberColumns, ...customColumns, 'error'];
+    const rows = errors.map(row => ({...toErrorReportRow(row), ...customFieldCells(row)}));
+    return serialize(rows, {columns});
+}
+
+// Compose the completion email for a finished import: the summary and its links,
+// plus the attached error report. Owns how the outcome is presented, so the
+// importer yields only the result and never touches email or CSV formatting.
+export default function buildCompletionEmail({result, recipient, labelName, importLabel, urlFor}: CompletionEmailInput): EmailPayload {
+    const siteUrl = new URL(urlFor('home', null, true));
+    const membersUrl = new URL('members', urlFor('admin', null, true));
+    if (importLabel) {
+        membersUrl.searchParams.set('label', importLabel.slug);
+    }
+
+    const html = emailTemplate({result, siteUrl, membersUrl, emailRecipient: recipient, importLabel});
+    const subject = result.imported > 0 ? 'Your member import is complete' : 'Your member import was unsuccessful';
+
+    return {
+        to: recipient,
+        subject,
+        html,
+        forceTextContent: true,
+        attachments: [{
+            filename: `${labelName} - Errors.csv`,
+            content: buildErrorReport(result.errors),
+            contentType: 'text/csv',
+            contentDisposition: 'attachment'
+        }]
+    };
+}

package/core/server/services/members/import-export/import/importer.js

@@ -0,0 +1,297 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const moment_timezone_1 = __importDefault(require("moment-timezone"));
+const completion_email_1 = __importDefault(require("./completion-email"));
+const csv_1 = require("../csv");
+const csv_2 = require("@tryghost/custom-field-types/csv");
+const metrics = require('@tryghost/metrics');
+const errors = require('@tryghost/errors');
+const tpl = require('@tryghost/tpl');
+const logging = require('@tryghost/logging');
+const messages = {
+    freeMemberNotAllowedImportTier: 'You cannot import a free member with a specified tier.',
+    invalidImportTier: '"{tier}" is not a valid tier.',
+    giftCannotCombineWithImportTier: 'Cannot specify both gift_id and import_tier.',
+    giftCannotCombineWithComplimentary: 'Cannot specify both gift_id and complimentary_plan.',
+    giftReassignFailed: 'Failed to reassign gift to member.'
+};
+// Columns whose presence makes a row slow to import (they reach out to Stripe), so
+// a file carrying any of them is deferred regardless of size. The routing decision
+// only ever sees the boolean this produces, never the column names themselves.
+const EXPENSIVE_COLUMNS = ['stripe_customer_id'];
+// The context each repository call runs under: the import identifies itself, and a
+// created member is additionally flagged as imported so downstream hooks can tell.
+const IMPORT_CONTEXT = { importer: true };
+const CREATE_CONTEXT = { import: true };
+// The import label members are tagged with, named for the request time. Shared by
+// the kernel (the label) and the deferred email (the error CSV filename).
+function buildImportLabelName(timezone) {
+    return `Import ${(0, moment_timezone_1.default)().tz(timezone).format('YYYY-MM-DD HH:mm')}`;
+}
+function hasExpensiveColumns(rows) {
+    return rows.some(row => EXPENSIVE_COLUMNS.some(column => !!row[column]));
+}
+// The routing decision, in terms it can reason about without knowing what a member
+// or a Stripe customer is: an empty or small, cheap file imports inline (an empty
+// one simply imports nothing); a large one, or one carrying expensive columns, is
+// too slow to hold a request open, so it is deferred to a background job.
+function canImportInline(rowCount, expensive, inlineThreshold) {
+    return rowCount <= inlineThreshold && !expensive;
+}
+class MembersCSVImporter {
+    _knex;
+    _readRows;
+    _spool;
+    _members;
+    _tiers;
+    _stripe;
+    _gifts;
+    _customFields;
+    _email;
+    _addJob;
+    _getTimezone;
+    _getInlineThreshold;
+    constructor({ knex, readRows, spool, members, tiers, stripe, gifts, customFields, email, addJob, getTimezone, getInlineThreshold }) {
+        this._knex = knex;
+        this._readRows = readRows;
+        this._spool = spool;
+        this._members = members;
+        this._tiers = tiers;
+        this._stripe = stripe;
+        this._gifts = gifts;
+        this._customFields = customFields;
+        this._email = email;
+        this._addJob = addJob;
+        this._getTimezone = getTimezone;
+        this._getInlineThreshold = getInlineThreshold;
+    }
+    async importCSV(request, verificationTrigger) {
+        const rows = await this._readRows(request.filePath, request.mapping);
+        const labelName = buildImportLabelName(this._getTimezone());
+        const extraLabels = request.extraLabels ?? [];
+        if (canImportInline(rows.length, hasExpensiveColumns(rows), this._getInlineThreshold())) {
+            const result = await this.importRows(rows, labelName, extraLabels);
+            await verificationTrigger.testImportThreshold();
+            return { deferred: false, originalImportSize: rows.length, result };
+        }
+        await this.deferImport(rows, { labelName, extraLabels, requestUserEmail: request.requestUserEmail }, verificationTrigger);
+        return { deferred: true, originalImportSize: rows.length };
+    }
+    async importInline(request, verificationTrigger) {
+        const rows = await this._readRows(request.filePath, request.mapping);
+        const result = await this.importRows(rows, buildImportLabelName(this._getTimezone()), request.extraLabels ?? []);
+        await verificationTrigger.testImportThreshold();
+        return result;
+    }
+    async deferImport(rows, { labelName, extraLabels, requestUserEmail }, verificationTrigger) {
+        // Resolved here, not at the API boundary, so the owner lookup only runs when
+        // a request without a user actually reaches the deferred path.
+        const emailRecipient = requestUserEmail ?? await this._email.getDefaultRecipient();
+        const spooled = await this._spool.write(rows);
+        this._addJob({
+            job: () => this.runImportJob(spooled, { labelName, extraLabels, emailRecipient }, verificationTrigger),
+            offloaded: false,
+            name: 'members-import'
+        });
+    }
+    // Swallows its own errors: a failed import or email must not leave the queued job
+    // rejected, and the verification trigger must still run afterwards either way.
+    async runImportJob(spooled, { labelName, extraLabels, emailRecipient }, verificationTrigger) {
+        try {
+            const spooledRows = await spooled.read();
+            const result = await this.importRows(spooledRows, labelName, extraLabels);
+            const importLabel = result.importLabel ?? null;
+            await this._email.send((0, completion_email_1.default)({
+                result,
+                recipient: emailRecipient,
+                labelName,
+                importLabel,
+                urlFor: this._email.urlFor
+            }));
+        }
+        catch (e) {
+            logging.error('Error in members import job');
+            logging.error(e);
+        }
+        finally {
+            await spooled.remove();
+        }
+        try {
+            await verificationTrigger.testImportThreshold();
+        }
+        catch (e) {
+            logging.error('Error in members import job when testing import threshold');
+            logging.error(e);
+        }
+    }
+    // The members import kernel, shared by the inline and deferred paths. Each row is
+    // created or updated in its own transaction, so one row's failure rolls back only
+    // itself; failures are collected, and any Stripe prices an import tier created are
+    // archived at the end.
+    async importRows(rows, labelName, extraLabels) {
+        const importLabel = { name: labelName };
+        const globalLabels = [importLabel, ...extraLabels];
+        const performStart = Date.now();
+        const defaultTier = await this._tiers.getDefault();
+        // The field set a custom_fields.* column is read against; empty when the feature
+        // is off, so a carried-through column is dropped and the write boundary stays shut.
+        const activeCustomFields = await this._customFields.activeFields();
+        const tierIdCache = new Map();
+        const archivableStripePriceIds = [];
+        // Copied per row: the member model stamps ids and trims names onto these in
+        // place, and each row runs in its own transaction that can roll back. A
+        // caller can hand in a nameless label, which the model would drop anyway.
+        const cloneGlobalLabels = () => globalLabels
+            .map(label => ({ ...label }))
+            .filter(label => label.name);
+        let imported = 0;
+        const importErrors = [];
+        for (const row of rows) {
+            let trx;
+            try {
+                if (row.gift_id) {
+                    if (row.import_tier) {
+                        throw wrapGiftError(new errors.DataImportError({ message: tpl(messages.giftCannotCombineWithImportTier) }));
+                    }
+                    if (row.complimentary_plan) {
+                        throw wrapGiftError(new errors.DataImportError({ message: tpl(messages.giftCannotCombineWithComplimentary) }));
+                    }
+                }
+                // Validate the row's custom field values before the transaction opens.
+                // planWrite only reads, and it throws on an invalid value to fail the row
+                // before any member write -- so there is no reason to hold a transaction
+                // across it, and doing so would deadlock the single-connection SQLite pool.
+                const customFieldPlan = activeCustomFields.length > 0
+                    ? await this._customFields.planWrite((0, csv_2.fieldValuesFromCsvRow)(activeCustomFields, row, csv_1.stripFormulaGuard))
+                    : [];
+                trx = await this._knex.transaction(undefined, { doNotRejectOnRollback: false });
+                const options = { transacting: trx, context: IMPORT_CONTEXT };
+                const createdAt = (row.created_at && (0, moment_timezone_1.default)(row.created_at).isAfter((0, moment_timezone_1.default)())) ? (0, moment_timezone_1.default)().toDate() : row.created_at;
+                const memberValues = {
+                    email: row.email,
+                    name: row.name,
+                    note: row.note,
+                    subscribed: row.subscribed,
+                    created_at: createdAt,
+                    labels: [...row.labels, ...cloneGlobalLabels()]
+                };
+                const existingMember = row.email
+                    ? await this._members.get({ email: row.email }, { ...options, withRelated: ['labels', 'newsletters'] })
+                    : null;
+                let member;
+                if (existingMember) {
+                    const existingLabels = existingMember.related('labels').toJSON();
+                    const existingNewsletters = existingMember.related('newsletters');
+                    if (existingNewsletters.length > 0 && memberValues.subscribed) {
+                        memberValues.newsletters = existingNewsletters.toJSON();
+                    }
+                    if (!existingNewsletters.length && memberValues.subscribed) {
+                        memberValues.subscribed = false;
+                    }
+                    if (!row.name) {
+                        memberValues.name = existingMember.name;
+                    }
+                    if (!row.note) {
+                        memberValues.note = existingMember.note;
+                    }
+                    member = await this._members.update({
+                        ...memberValues,
+                        labels: existingLabels.concat(memberValues.labels)
+                    }, { ...options, id: existingMember.id });
+                }
+                else {
+                    member = await this._members.create(memberValues, Object.assign({}, options, { context: CREATE_CONTEXT }));
+                }
+                let importTierId;
+                if (row.import_tier) {
+                    if (!tierIdCache.has(row.import_tier)) {
+                        const tier = await this._tiers.getByName(row.import_tier);
+                        tierIdCache.set(row.import_tier, tier ? tier.id.toString() : null);
+                    }
+                    importTierId = tierIdCache.get(row.import_tier);
+                    if (!importTierId) {
+                        throw new errors.DataImportError({ message: tpl(messages.invalidImportTier, { tier: row.import_tier }) });
+                    }
+                }
+                if (row.stripe_customer_id) {
+                    let stripeCustomerId;
+                    if (row.stripe_customer_id.toLowerCase() === 'auto') {
+                        stripeCustomerId = row.email ? await this._members.getCustomerIdByEmail(row.email) : undefined;
+                    }
+                    else {
+                        stripeCustomerId = row.stripe_customer_id;
+                    }
+                    if (stripeCustomerId) {
+                        if (row.import_tier) {
+                            const { isNewStripePrice, stripePriceId } = await this._stripe.forceStripeSubscriptionToProduct({
+                                customer_id: stripeCustomerId,
+                                product_id: importTierId
+                            }, options);
+                            if (isNewStripePrice) {
+                                archivableStripePriceIds.push(stripePriceId);
+                            }
+                        }
+                        await this._members.linkStripeCustomer({ customer_id: stripeCustomerId, member_id: member.id }, options);
+                    }
+                }
+                else if (row.complimentary_plan) {
+                    const products = [];
+                    if (row.import_tier) {
+                        products.push({ id: importTierId });
+                    }
+                    else {
+                        products.push({ id: defaultTier.id.toString() });
+                    }
+                    await this._members.update({ products }, { ...options, id: member.id });
+                }
+                else if (row.import_tier) {
+                    throw new errors.DataImportError({ message: tpl(messages.freeMemberNotAllowedImportTier) });
+                }
+                if (row.gift_id) {
+                    try {
+                        await this._gifts.reassignRedeemer(row.gift_id, member.id, { transacting: trx });
+                    }
+                    catch (giftError) {
+                        throw wrapGiftError(giftError);
+                    }
+                }
+                // On the row's transaction, so the values commit or roll back with the member.
+                await this._customFields.applyWrite(member.id, customFieldPlan, trx);
+                await trx.commit();
+                imported += 1;
+            }
+            catch (error) {
+                const errorList = Array.isArray(error) ? error : [error];
+                const errorMessage = errorList
+                    .map(e => (typeof e === 'object' && e !== null && 'message' in e ? e.message : undefined))
+                    .join(', ');
+                // trx is unset if the row failed before the transaction opened (a bad
+                // custom field value or gift combination).
+                if (trx) {
+                    await trx.rollback();
+                }
+                importErrors.push({ ...row, error: errorMessage });
+            }
+        }
+        await Promise.all(archivableStripePriceIds.map(id => this._stripe.archivePrice(id)));
+        metrics.metric('members-import', { imported, errors: importErrors.length, value: Date.now() - performStart });
+        if (imported > 0) {
+            // The import label exists now that members carry it, so fetch it to report.
+            // Guarded against a null lookup in case a row is ever counted without it.
+            const importLabelModel = await this._members.getImportLabel(labelName);
+            return { imported, errors: importErrors, importLabel: importLabelModel?.toJSON() };
+        }
+        return { imported: 0, errors: importErrors };
+    }
+}
+function wrapGiftError(error) {
+    const err = error;
+    const message = (err && typeof err.message === 'string' && err.message) || tpl(messages.giftReassignFailed);
+    return new errors.DataImportError({
+        message: `Member cannot be assigned to a gift: ${message}`
+    });
+}
+exports.default = MembersCSVImporter;