ghost 6.53.0 → 6.54.1

package/core/server/services/members/import-export/import/importer.js

@@ -0,0 +1,276 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const moment_timezone_1 = __importDefault(require("moment-timezone"));
+const completion_email_1 = __importDefault(require("./completion-email"));
+const metrics = require('@tryghost/metrics');
+const errors = require('@tryghost/errors');
+const tpl = require('@tryghost/tpl');
+const logging = require('@tryghost/logging');
+const messages = {
+    freeMemberNotAllowedImportTier: 'You cannot import a free member with a specified tier.',
+    invalidImportTier: '"{tier}" is not a valid tier.',
+    giftCannotCombineWithImportTier: 'Cannot specify both gift_id and import_tier.',
+    giftCannotCombineWithComplimentary: 'Cannot specify both gift_id and complimentary_plan.',
+    giftReassignFailed: 'Failed to reassign gift to member.'
+};
+// Columns whose presence makes a row slow to import (they reach out to Stripe), so
+// a file carrying any of them is deferred regardless of size. The routing decision
+// only ever sees the boolean this produces, never the column names themselves.
+const EXPENSIVE_COLUMNS = ['stripe_customer_id'];
+// The context each repository call runs under: the import identifies itself, and a
+// created member is additionally flagged as imported so downstream hooks can tell.
+const IMPORT_CONTEXT = { importer: true };
+const CREATE_CONTEXT = { import: true };
+// The import label members are tagged with, named for the request time. Shared by
+// the kernel (the label) and the deferred email (the error CSV filename).
+function buildImportLabelName(timezone) {
+    return `Import ${(0, moment_timezone_1.default)().tz(timezone).format('YYYY-MM-DD HH:mm')}`;
+}
+function hasExpensiveColumns(rows) {
+    return rows.some(row => EXPENSIVE_COLUMNS.some(column => !!row[column]));
+}
+// The routing decision, in terms it can reason about without knowing what a member
+// or a Stripe customer is: an empty or small, cheap file imports inline (an empty
+// one simply imports nothing); a large one, or one carrying expensive columns, is
+// too slow to hold a request open, so it is deferred to a background job.
+function canImportInline(rowCount, expensive, inlineThreshold) {
+    return rowCount <= inlineThreshold && !expensive;
+}
+class MembersCSVImporter {
+    _knex;
+    _readRows;
+    _spool;
+    _members;
+    _tiers;
+    _stripe;
+    _gifts;
+    _email;
+    _addJob;
+    _getTimezone;
+    _getInlineThreshold;
+    constructor({ knex, readRows, spool, members, tiers, stripe, gifts, email, addJob, getTimezone, getInlineThreshold }) {
+        this._knex = knex;
+        this._readRows = readRows;
+        this._spool = spool;
+        this._members = members;
+        this._tiers = tiers;
+        this._stripe = stripe;
+        this._gifts = gifts;
+        this._email = email;
+        this._addJob = addJob;
+        this._getTimezone = getTimezone;
+        this._getInlineThreshold = getInlineThreshold;
+    }
+    async importCSV(request, verificationTrigger) {
+        const rows = await this._readRows(request.filePath, request.mapping);
+        const labelName = buildImportLabelName(this._getTimezone());
+        const extraLabels = request.extraLabels ?? [];
+        if (canImportInline(rows.length, hasExpensiveColumns(rows), this._getInlineThreshold())) {
+            const result = await this.importRows(rows, labelName, extraLabels);
+            await verificationTrigger.testImportThreshold();
+            return { deferred: false, originalImportSize: rows.length, result };
+        }
+        await this.deferImport(rows, { labelName, extraLabels, requestUserEmail: request.requestUserEmail }, verificationTrigger);
+        return { deferred: true, originalImportSize: rows.length };
+    }
+    async importInline(request, verificationTrigger) {
+        const rows = await this._readRows(request.filePath, request.mapping);
+        const result = await this.importRows(rows, buildImportLabelName(this._getTimezone()), request.extraLabels ?? []);
+        await verificationTrigger.testImportThreshold();
+        return result;
+    }
+    async deferImport(rows, { labelName, extraLabels, requestUserEmail }, verificationTrigger) {
+        // Resolved here, not at the API boundary, so the owner lookup only runs when
+        // a request without a user actually reaches the deferred path.
+        const emailRecipient = requestUserEmail ?? await this._email.getDefaultRecipient();
+        const spooled = await this._spool.write(rows);
+        this._addJob({
+            job: () => this.runImportJob(spooled, { labelName, extraLabels, emailRecipient }, verificationTrigger),
+            offloaded: false,
+            name: 'members-import'
+        });
+    }
+    // Swallows its own errors: a failed import or email must not leave the queued job
+    // rejected, and the verification trigger must still run afterwards either way.
+    async runImportJob(spooled, { labelName, extraLabels, emailRecipient }, verificationTrigger) {
+        try {
+            const spooledRows = await spooled.read();
+            const result = await this.importRows(spooledRows, labelName, extraLabels);
+            const importLabel = result.importLabel ?? null;
+            await this._email.send((0, completion_email_1.default)({
+                result,
+                recipient: emailRecipient,
+                labelName,
+                importLabel,
+                urlFor: this._email.urlFor
+            }));
+        }
+        catch (e) {
+            logging.error('Error in members import job');
+            logging.error(e);
+        }
+        finally {
+            await spooled.remove();
+        }
+        try {
+            await verificationTrigger.testImportThreshold();
+        }
+        catch (e) {
+            logging.error('Error in members import job when testing import threshold');
+            logging.error(e);
+        }
+    }
+    // The members import kernel, shared by the inline and deferred paths. Each row is
+    // created or updated in its own transaction, so one row's failure rolls back only
+    // itself; failures are collected, and any Stripe prices an import tier created are
+    // archived at the end.
+    async importRows(rows, labelName, extraLabels) {
+        const importLabel = { name: labelName };
+        const globalLabels = [importLabel, ...extraLabels];
+        const performStart = Date.now();
+        const defaultTier = await this._tiers.getDefault();
+        const tierIdCache = new Map();
+        const archivableStripePriceIds = [];
+        // Copied per row: the member model stamps ids and trims names onto these in
+        // place, and each row runs in its own transaction that can roll back. A
+        // caller can hand in a nameless label, which the model would drop anyway.
+        const cloneGlobalLabels = () => globalLabels
+            .map(label => ({ ...label }))
+            .filter(label => label.name);
+        let imported = 0;
+        const importErrors = [];
+        for (const row of rows) {
+            const trx = await this._knex.transaction(undefined, { doNotRejectOnRollback: false });
+            const options = { transacting: trx, context: IMPORT_CONTEXT };
+            try {
+                if (row.gift_id) {
+                    if (row.import_tier) {
+                        throw wrapGiftError(new errors.DataImportError({ message: tpl(messages.giftCannotCombineWithImportTier) }));
+                    }
+                    if (row.complimentary_plan) {
+                        throw wrapGiftError(new errors.DataImportError({ message: tpl(messages.giftCannotCombineWithComplimentary) }));
+                    }
+                }
+                const createdAt = (row.created_at && (0, moment_timezone_1.default)(row.created_at).isAfter((0, moment_timezone_1.default)())) ? (0, moment_timezone_1.default)().toDate() : row.created_at;
+                const memberValues = {
+                    email: row.email,
+                    name: row.name,
+                    note: row.note,
+                    subscribed: row.subscribed,
+                    created_at: createdAt,
+                    labels: [...row.labels, ...cloneGlobalLabels()]
+                };
+                const existingMember = row.email
+                    ? await this._members.get({ email: row.email }, { ...options, withRelated: ['labels', 'newsletters'] })
+                    : null;
+                let member;
+                if (existingMember) {
+                    const existingLabels = existingMember.related('labels').toJSON();
+                    const existingNewsletters = existingMember.related('newsletters');
+                    if (existingNewsletters.length > 0 && memberValues.subscribed) {
+                        memberValues.newsletters = existingNewsletters.toJSON();
+                    }
+                    if (!existingNewsletters.length && memberValues.subscribed) {
+                        memberValues.subscribed = false;
+                    }
+                    if (!row.name) {
+                        memberValues.name = existingMember.name;
+                    }
+                    if (!row.note) {
+                        memberValues.note = existingMember.note;
+                    }
+                    member = await this._members.update({
+                        ...memberValues,
+                        labels: existingLabels.concat(memberValues.labels)
+                    }, { ...options, id: existingMember.id });
+                }
+                else {
+                    member = await this._members.create(memberValues, Object.assign({}, options, { context: CREATE_CONTEXT }));
+                }
+                let importTierId;
+                if (row.import_tier) {
+                    if (!tierIdCache.has(row.import_tier)) {
+                        const tier = await this._tiers.getByName(row.import_tier);
+                        tierIdCache.set(row.import_tier, tier ? tier.id.toString() : null);
+                    }
+                    importTierId = tierIdCache.get(row.import_tier);
+                    if (!importTierId) {
+                        throw new errors.DataImportError({ message: tpl(messages.invalidImportTier, { tier: row.import_tier }) });
+                    }
+                }
+                if (row.stripe_customer_id) {
+                    let stripeCustomerId;
+                    if (row.stripe_customer_id.toLowerCase() === 'auto') {
+                        stripeCustomerId = row.email ? await this._members.getCustomerIdByEmail(row.email) : undefined;
+                    }
+                    else {
+                        stripeCustomerId = row.stripe_customer_id;
+                    }
+                    if (stripeCustomerId) {
+                        if (row.import_tier) {
+                            const { isNewStripePrice, stripePriceId } = await this._stripe.forceStripeSubscriptionToProduct({
+                                customer_id: stripeCustomerId,
+                                product_id: importTierId
+                            }, options);
+                            if (isNewStripePrice) {
+                                archivableStripePriceIds.push(stripePriceId);
+                            }
+                        }
+                        await this._members.linkStripeCustomer({ customer_id: stripeCustomerId, member_id: member.id }, options);
+                    }
+                }
+                else if (row.complimentary_plan) {
+                    const products = [];
+                    if (row.import_tier) {
+                        products.push({ id: importTierId });
+                    }
+                    else {
+                        products.push({ id: defaultTier.id.toString() });
+                    }
+                    await this._members.update({ products }, { ...options, id: member.id });
+                }
+                else if (row.import_tier) {
+                    throw new errors.DataImportError({ message: tpl(messages.freeMemberNotAllowedImportTier) });
+                }
+                if (row.gift_id) {
+                    try {
+                        await this._gifts.reassignRedeemer(row.gift_id, member.id, { transacting: trx });
+                    }
+                    catch (giftError) {
+                        throw wrapGiftError(giftError);
+                    }
+                }
+                await trx.commit();
+                imported += 1;
+            }
+            catch (error) {
+                const errorList = Array.isArray(error) ? error : [error];
+                const errorMessage = errorList
+                    .map(e => (typeof e === 'object' && e !== null && 'message' in e ? e.message : undefined))
+                    .join(', ');
+                await trx.rollback();
+                importErrors.push({ ...row, error: errorMessage });
+            }
+        }
+        await Promise.all(archivableStripePriceIds.map(id => this._stripe.archivePrice(id)));
+        metrics.metric('members-import', { imported, errors: importErrors.length, value: Date.now() - performStart });
+        if (imported > 0) {
+            // The import label exists now that members carry it, so fetch it to report.
+            // Guarded against a null lookup in case a row is ever counted without it.
+            const importLabelModel = await this._members.getImportLabel(labelName);
+            return { imported, errors: importErrors, importLabel: importLabelModel?.toJSON() };
+        }
+        return { imported: 0, errors: importErrors };
+    }
+}
+function wrapGiftError(error) {
+    const err = error;
+    const message = (err && typeof err.message === 'string' && err.message) || tpl(messages.giftReassignFailed);
+    return new errors.DataImportError({
+        message: `Member cannot be assigned to a gift: ${message}`
+    });
+}
+exports.default = MembersCSVImporter;