ghost 6.53.0 → 6.54.0

package/core/server/services/members/importer/members-csv-importer.js

@@ -2,7 +2,7 @@ const moment = require('moment-timezone');
 const path = require('path');
 const fs = require('fs-extra');
 const metrics = require('@tryghost/metrics');
-const membersCSV = require('@tryghost/members-csv');
+const membersCSV = require('../csv');
 const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 const emailTemplate = require('./email-template');
@@ -61,9 +61,10 @@ module.exports = class MembersCSVImporter {
     /**
      * @param {MembersCSVImporterOptions} options
      */
-    constructor({storagePath, getTimezone, getMembersRepository, getDefaultTier, getTierByName, getGiftService, sendEmail, isSet, addJob, knex, urlFor, context, stripeUtils}) {
+    constructor({storagePath, getTimezone, getInlineThreshold, getMembersRepository, getDefaultTier, getTierByName, getGiftService, sendEmail, isSet, addJob, knex, urlFor, context, stripeUtils}) {
         this._storagePath = storagePath;
         this._getTimezone = getTimezone;
+        this._getInlineThreshold = getInlineThreshold;
         this._getMembersRepository = getMembersRepository;
         this._getDefaultTier = getDefaultTier;
         this._getTierByName = getTierByName;
@@ -87,11 +88,10 @@ module.exports = class MembersCSVImporter {
      *
      * @param {string} inputFilePath - The path to the CSV to prepare
      * @param {Object.<string, string>} [headerMapping] - An object whose keys are headers in the input CSV and values are the header to replace it with
-     * @param {Array<string>} [defaultLabels] - A list of labels to apply to every member
      *
      * @returns {Promise<{filePath: string, batches: number, metadata: Object.<string, any>}>} - A promise resolving to the data including filePath of "prepared" CSV
      */
-    async prepare(inputFilePath, headerMapping, defaultLabels) {
+    async prepare(inputFilePath, headerMapping) {
         headerMapping = headerMapping || DEFAULT_CSV_HEADER_MAPPING;
         // @NOTE: investigate why is it "1" and do we even need this concept anymore?
         const batchSize = 1;
@@ -108,10 +108,16 @@ module.exports = class MembersCSVImporter {
         }
 
         // completely rely on explicit user input for header mappings
-        const rows = await membersCSV.parse(inputFilePath, headerMapping, defaultLabels);
-        const columns = Object.keys(rows[0]);
+        const rows = await membersCSV.parse(inputFilePath, headerMapping);
+        // Columns come from every row, not the first. A row with fewer fields than
+        // the header parses to fewer keys, so reading the schema off one row drops
+        // the columns it happened to omit for the whole file. A file of headers and
+        // no rows has no columns, and serialising nothing has nothing to say.
+        const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
         const numberOfBatches = Math.ceil(rows.length / batchSize);
-        const mappedCSV = membersCSV.unparse(rows, columns);
+        // No formula escaping: this file is only ever read back by the import, not
+        // opened by a person, and escaping would be stored on the member verbatim.
+        const mappedCSV = columns.length ? membersCSV.unparse(rows, columns, {escapeFormulae: false}) : '';
 
         const hasStripeData = !!(rows.find(function rowHasStripeData(row) {
             return !!row.stripe_customer_id;
@@ -133,8 +139,17 @@ module.exports = class MembersCSVImporter {
      *
      * @param {string} filePath - the path to a "prepared" CSV file
      */
-    async perform(filePath) {
+    async perform(filePath, globalLabels = []) {
         const performStart = Date.now();
+        // Copied per row, not shared: the member model stamps ids and trims names onto
+        // label objects in place, one of these is the caller's import label that
+        // process() looks up again once the import finishes, and each row runs in its
+        // own transaction that can roll back.
+        const cloneGlobalLabels = () => (globalLabels || [])
+            .map(label => (typeof label === 'string' ? {name: label} : {...label}))
+            // A caller can hand in a nameless label; the model would drop it, but the
+            // round-trip this replaced dropped it before the model ever saw it.
+            .filter(label => label.name);
         const rows = await membersCSV.parse(filePath, DEFAULT_CSV_HEADER_MAPPING);
 
         const defaultTier = await this._getDefaultTier();
@@ -182,7 +197,10 @@ module.exports = class MembersCSVImporter {
                     note: row.note,
                     subscribed: row.subscribed,
                     created_at: createdAt,
-                    labels: row.labels
+                    // Form labels are applied here rather than folded into the parsed
+                    // rows, so a name that contains a comma does not go through the
+                    // prepared file's comma-separated labels column and split in two.
+                    labels: [...row.labels, ...cloneGlobalLabels()]
                 };
                 const existingMember = await membersRepository.get({email: memberValues.email}, {
                     ...options,
@@ -419,12 +437,12 @@ module.exports = class MembersCSVImporter {
      */
     async process({pathToCSV, headerMapping, globalLabels, importLabel, user, LabelModel, forceInline, verificationTrigger}) {
         const meta = {};
-        const job = await this.prepare(pathToCSV, headerMapping, globalLabels);
+        const job = await this.prepare(pathToCSV, headerMapping);
 
         meta.originalImportSize = job.batches;
 
-        if ((job.batches <= 500 && !job.metadata.hasStripeData) || forceInline) {
-            const result = await this.perform(job.filePath);
+        if ((job.batches <= this._getInlineThreshold() && !job.metadata.hasStripeData) || forceInline) {
+            const result = await this.perform(job.filePath, globalLabels);
             const importLabelModel = result.imported ? await LabelModel.findOne(importLabel) : null;
             await verificationTrigger.testImportThreshold();
 
@@ -442,7 +460,7 @@ module.exports = class MembersCSVImporter {
             this._addJob({
                 job: async () => {
                     try {
-                        const result = await this.perform(job.filePath);
+                        const result = await this.perform(job.filePath, globalLabels);
                         const importLabelModel = result.imported ? await LabelModel.findOne(importLabel) : null;
                         const emailContent = this.generateCompletionEmail(result, {
                             emailRecipient,

package/core/server/services/members/members-api/members-api.js

@@ -81,10 +81,11 @@ module.exports = function MembersAPI({
     sentry,
     settingsHelpers,
     urlUtils,
+    urlService,
     commentsService,
     emailAddressService,
     giftService,
-    customFieldsService
+    customFieldValues
 }) {
     const tokenService = new TokenService({
         privateKey,
@@ -123,6 +124,7 @@ module.exports = function MembersAPI({
     });
 
     const eventRepository = new EventRepository({
+        urlService,
         DonationPaymentEvent,
         EmailRecipient,
         MemberSubscribeEvent,
@@ -167,7 +169,7 @@ module.exports = function MembersAPI({
         nextPaymentCalculator,
         commentsService,
         giftService,
-        customFieldsService
+        customFieldValues
     });
 
     const geolocationService = new GeolocationService();

package/core/server/services/members/members-api/repositories/event-repository.js

@@ -34,6 +34,7 @@ module.exports = class EventRepository {
         Comment,
         labsService,
         memberAttributionService,
+        urlService,
         MemberEmailChangeEvent,
         AutomatedEmailRecipient,
         Gift
@@ -47,6 +48,7 @@ module.exports = class EventRepository {
         this._EmailRecipient = EmailRecipient;
         this._Comment = Comment;
         this._labsService = labsService;
+        this._urlService = urlService;
         this._MemberCreatedEvent = MemberCreatedEvent;
         this._SubscriptionCreatedEvent = SubscriptionCreatedEvent;
         this._MemberLinkClickEvent = MemberLinkClickEvent;
@@ -59,6 +61,12 @@ module.exports = class EventRepository {
         this._knex = db.knex;
     }
 
+    // the URL service needs the attributed post's relations (e.g. tags for a
+    // tag-filtered collection) to build its URL; prefix them onto the eager-load
+    #attributionPostRelations(prefix) {
+        return this._urlService.getRequiredRelations().map(relation => `${prefix}.${relation}`);
+    }
+
     async getEventTimeline(options = {}) {
         if (!options.limit) {
             options.limit = 10;
@@ -213,6 +221,7 @@ module.exports = class EventRepository {
             withRelated: [
                 'member',
                 'subscriptionCreatedEvent.postAttribution',
+                ...this.#attributionPostRelations('subscriptionCreatedEvent.postAttribution'),
                 'subscriptionCreatedEvent.userAttribution',
                 'subscriptionCreatedEvent.tagAttribution',
                 'subscriptionCreatedEvent.memberCreatedEvent',
@@ -343,6 +352,7 @@ module.exports = class EventRepository {
             withRelated: [
                 'member',
                 'postAttribution',
+                ...this.#attributionPostRelations('postAttribution'),
                 'userAttribution',
                 'tagAttribution',
                 'signupStatusEvent'
@@ -402,6 +412,7 @@ module.exports = class EventRepository {
             withRelated: [
                 'member',
                 'postAttribution',
+                ...this.#attributionPostRelations('postAttribution'),
                 'userAttribution',
                 'tagAttribution'
             ],
@@ -591,7 +602,7 @@ module.exports = class EventRepository {
     async getCommentEvents(options = {}, filter) {
         options = {
             ...options,
-            withRelated: ['member', 'post', 'post.tags', 'post.authors', 'parent'],
+            withRelated: ['member', 'post', ...this.#attributionPostRelations('post'), 'parent'],
             filter: 'member_id:-null+custom:true',
             useBasicCount: true,
             mongoTransformer: chainTransformers(
@@ -625,7 +636,7 @@ module.exports = class EventRepository {
     async getClickEvents(options = {}, filter) {
         options = {
             ...options,
-            withRelated: ['member', 'link', 'link.post', 'link.post.tags', 'link.post.authors'],
+            withRelated: ['member', 'link', 'link.post', ...this.#attributionPostRelations('link.post')],
             filter: 'custom:true',
             useBasicCount: true,
             mongoTransformer: chainTransformers(

package/core/server/services/members/members-api/services/member-bread-service.js

@@ -6,7 +6,8 @@ const moment = require('moment');
 const messages = {
     stripeNotConnected: 'Missing Stripe connection.',
     memberAlreadyExists: 'Member already exists.',
-    memberNotFound: 'Member not found.'
+    memberNotFound: 'Member not found.',
+    customFieldsOnAdd: 'Custom field values cannot be set while creating a member. Create the member, then set values with an edit.'
 };
 
 // Stored in the action's `context.action_name`; Admin maps it to a display label.
@@ -36,15 +37,6 @@ const CUSTOM_FIELDS_EDITED_ACTION = 'custom_fields_edited';
  * @prop {{getActiveByMembers: (memberIds: string[]) => Promise<Map<string, {cadence: 'month' | 'year', currency: string, amount: number}>>}} service
  */
 
-/**
- * @typedef {object} ICustomFieldsServiceWrapper
- * @prop {{
- *   getValuesForMembers: (memberIds: string[]) => Promise<Map<string, Record<string, unknown>>>,
- *   planWrite: (values: unknown) => Promise<object[]>,
- *   applyWrite: (memberId: string, writes: object[]) => Promise<void>
- * }} values
- */
-
 module.exports = class MemberBREADService {
     /**
      * @param {object} deps
@@ -58,9 +50,9 @@ module.exports = class MemberBREADService {
      * @param {import('../../../settings-helpers/settings-helpers')} deps.settingsHelpers
      * @param {import('./next-payment-calculator')} deps.nextPaymentCalculator
      * @param {IGiftServiceWrapper} deps.giftService
-     * @param {ICustomFieldsServiceWrapper} deps.customFieldsService
+     * @param {import('../../../members-custom-fields/values-service').CustomFieldValuesService} deps.customFieldValues Required: boot builds it before the members service
      */
-    constructor({memberRepository, labsService, emailService, stripeService, offersAPI, memberAttributionService, emailSuppressionList, settingsHelpers, nextPaymentCalculator, commentsService, giftService, customFieldsService}) {
+    constructor({memberRepository, labsService, emailService, stripeService, offersAPI, memberAttributionService, emailSuppressionList, settingsHelpers, nextPaymentCalculator, commentsService, giftService, customFieldValues}) {
         this.offersAPI = offersAPI;
         /** @private */
         this.memberRepository = memberRepository;
@@ -83,7 +75,7 @@ module.exports = class MemberBREADService {
         /** @private */
         this.giftService = giftService;
         /** @private */
-        this.customFieldsService = customFieldsService;
+        this.customFieldValues = customFieldValues;
     }
 
     /**
@@ -112,7 +104,7 @@ module.exports = class MemberBREADService {
             return null;
         }
 
-        return this.customFieldsService.values.getValuesForMembers(memberIds);
+        return this.customFieldValues.getValuesForMembers(memberIds);
     }
 
     /**
@@ -444,7 +436,36 @@ module.exports = class MemberBREADService {
         return member;
     }
 
+    /**
+     * @private
+     * The write-side flag gate, paired with `fetchCustomFieldValues` on the read
+     * side. The schema declares `custom_fields` for every site, so the key arrives
+     * whether or not the feature is on and this is what decides it goes no further.
+     * @param {object} data
+     */
+    dropCustomFieldsWhenDisabled(data) {
+        if (!this.labsService.isSet('membersCustomFields')) {
+            delete data.custom_fields;
+        }
+    }
+
     async add(data, options) {
+        this.dropCustomFieldsWhenDisabled(data);
+
+        // Values cannot be set on create, only on a subsequent edit. `namesValues`
+        // both judges the body and rejects a malformed one, so a body refused here is
+        // refused on edit for the same reason.
+        if (this.customFieldValues.namesValues(data.custom_fields)) {
+            throw new errors.ValidationError({
+                message: tpl(messages.customFieldsOnAdd),
+                property: 'custom_fields'
+            });
+        }
+
+        // Not a member column, so it comes off before the repository sees it. Only
+        // an absent key or one naming no values gets this far.
+        delete data.custom_fields;
+
         if (!this.stripeService.configured && (data.comped || data.stripe_customer_id)) {
             const property = data.comped ? 'comped' : 'stripe_customer_id';
             throw new errors.ValidationError({
@@ -527,10 +548,12 @@ module.exports = class MemberBREADService {
     async edit(data, options) {
         delete data.last_seen_at;
 
+        this.dropCustomFieldsWhenDisabled(data);
+
         // Values live in their own table, so they come off the member data before
-        // the repository sees it — `custom_fields` is not a member column. It only
-        // reaches here at all when the flag is on (the input validator strips it
-        // otherwise), so its presence is the signal to write.
+        // the repository sees it — `custom_fields` is not a member column. The gate
+        // above has already dropped it when the feature is off, so by this point its
+        // presence is the signal to write.
         const customFields = data.custom_fields;
         const writeCustomFields = customFields !== undefined;
         delete data.custom_fields;
@@ -540,7 +563,7 @@ module.exports = class MemberBREADService {
         // plan to apply once below, so the values aren't resolved and validated
         // twice.
         const plannedCustomFields = writeCustomFields
-            ? await this.customFieldsService.values.planWrite(customFields)
+            ? await this.customFieldValues.planWrite(customFields)
             : null;
 
         let model;
@@ -589,7 +612,7 @@ module.exports = class MemberBREADService {
         }
 
         if (plannedCustomFields) {
-            await this.customFieldsService.values.applyWrite(model.id, plannedCustomFields);
+            await this.customFieldValues.applyWrite(model.id, plannedCustomFields);
 
             // Custom fields aren't a member column or relation, so an edit touching
             // only them leaves `model._changed` empty and the save fires nothing.

package/core/server/services/members/middleware.js

@@ -4,7 +4,7 @@ const logging = require('@tryghost/logging');
 const membersService = require('./service');
 const emailSuppressionList = require('../email-suppression-list');
 const models = require('../../models');
-const urlUtils = require('../../../shared/url-utils');
+const urlUtils = require('../../../shared/url-utils').default;
 const spamPrevention = require('../../web/shared/middleware/api/spam-prevention');
 const {
     formattedMemberResponse,

package/core/server/services/members/service.js

@@ -5,10 +5,11 @@ const MembersSSR = require('./members-ssr');
 const db = require('../../data/db');
 const MembersConfigProvider = require('./members-config-provider');
 const makeMembersCSVImporter = require('./importer');
+const {resolveInlineThreshold} = require('./importer/config');
 const MembersStats = require('./stats/members-stats');
 const memberJobs = require('./jobs');
 const logging = require('@tryghost/logging');
-const urlUtils = require('../../../shared/url-utils');
+const urlUtils = require('../../../shared/url-utils').default;
 const labsService = require('../../../shared/labs');
 const settingsCache = require('../../../shared/settings-cache');
 const config = require('../../../shared/config');
@@ -50,6 +51,9 @@ const initMembersCSVImporter = ({stripeAPIService}) => {
     return makeMembersCSVImporter({
         storagePath: config.getContentPath('data'),
         getTimezone: () => settingsCache.get('timezone'),
+        // A getter rather than a value because the threshold is an operator
+        // setting that can change between requests
+        getInlineThreshold: () => resolveInlineThreshold(config.get('members:importer:inlineThreshold')),
         getMembersRepository: async () => {
             const api = await module.exports.api;
             return api.members;
@@ -151,6 +155,14 @@ module.exports = {
         module.exports.verificationTrigger = verificationTrigger;
 
         const membersCSVImporter = initMembersCSVImporter({stripeAPIService: stripeService.api});
+        // Constructed here rather than required statically: the exporter needs the
+        // custom fields services, which boot builds before this one.
+        const customFields = require('../members-custom-fields');
+        module.exports.export = require('./exporter/query')({
+            definitions: customFields.definitions,
+            values: customFields.values
+        });
+
         module.exports.processImport = async (options) => {
             return await membersCSVImporter.process({...options, verificationTrigger});
         };
@@ -196,7 +208,8 @@ module.exports = {
     processImport: null,
 
     stats: membersStats,
-    export: require('./exporter/query')
+
+    export: null
 };
 
 module.exports.middleware = require('./middleware');

package/core/server/services/members/stripe-connect.js

@@ -4,7 +4,7 @@ const {Buffer} = require('buffer');
 const {randomBytes} = require('crypto');
 
 const config = require('../../../shared/config');
-const urlUtils = require('../../../shared/url-utils');
+const urlUtils = require('../../../shared/url-utils').default;
 
 const messages = {
     incorrectState: 'State did not match.'

package/core/server/services/members-custom-fields/actions.js

@@ -24,6 +24,9 @@ async function recordCustomFieldAction({ Action, context, verb, subject, details
         await Action.add({
             event: COMMANDS[verb],
             resource_type: 'member_custom_field',
+            // The field's id: this column holds 24 characters, and a key minted from
+            // a publisher-chosen name is bounded by the far wider key column, so only
+            // the id fits every field.
             resource_id: subject,
             actor_type: context.actor.type,
             actor_id: context.actor.id,

package/core/server/services/members-custom-fields/actions.ts

@@ -29,8 +29,9 @@ export type CustomFieldVerb = keyof typeof COMMANDS;
 // `details` is stored in the action's `context` column (Ghost's slot for "diffs,
 // meta"). We always pass the field's `primary_name` so the log reads as a human
 // label, not a bare key — this is what keeps the timeline legible after a hard
-// delete, when the row itself is gone.
-export type CustomFieldActionDetails = {primary_name: string; previous_name?: string};
+// delete, when the row itself is gone. `key` rides alongside it because the key is
+// how a field is addressed publicly — its id never leaves the API.
+export type CustomFieldActionDetails = {primary_name: string; key: string; previous_name?: string};
 
 export type RecordCustomFieldAction =
     (input: {context: RequestContext; verb: CustomFieldVerb; subject: string; details: CustomFieldActionDetails}) => Promise<void>;
@@ -47,6 +48,9 @@ export async function recordCustomFieldAction(
         await Action.add({
             event: COMMANDS[verb],
             resource_type: 'member_custom_field',
+            // The field's id: this column holds 24 characters, and a key minted from
+            // a publisher-chosen name is bounded by the far wider key column, so only
+            // the id fits every field.
             resource_id: subject,
             actor_type: context.actor.type,
             actor_id: context.actor.id,

package/core/server/services/members-custom-fields/config.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveMaxDefinitions = resolveMaxDefinitions;
+const zod_1 = require("zod");
+/**
+ * Resolving Ghost's config into the values this service runs on.
+ *
+ * Config arrives untyped: defaults.json supplies the shipped value, and any
+ * higher layer — an operator's config file, an environment variable, argv — can
+ * replace it with anything at all. Turning that into something usable is a
+ * config concern, so it happens here, at the boundary. The service is handed an
+ * answer and never learns where it came from.
+ */
+// A ceiling in the form the service needs it. A numeric string is accepted
+// because a value supplied through an environment variable can arrive as one.
+const Ceiling = zod_1.z
+    .union([zod_1.z.number(), zod_1.z.string().trim().min(1)])
+    .transform(Number)
+    .pipe(zod_1.z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER));
+// defaults.json is the single source of the shipped ceiling: it is the lowest
+// config layer and the file an operator actually reads and edits, so it is read
+// rather than restated. Parsed eagerly because a missing or malformed value
+// there is our bug rather than an operator's, and should fail loudly at boot
+// instead of quietly becoming the fallback for everything below.
+const shippedCeiling = Ceiling.parse(require('../../../shared/config/defaults.json').members.customFields.maxDefinitions);
+// A higher layer can override the shipped value with anything. Something
+// unreadable falls back to the shipped ceiling rather than throwing, because an
+// operator can correct this live and a bad value must not take the feature down
+// with it — and rather than meaning "no ceiling", because a typo must not
+// silently remove the safeguard.
+const MaxDefinitions = Ceiling.catch(shippedCeiling);
+function resolveMaxDefinitions(configured) {
+    return MaxDefinitions.parse(configured);
+}

package/core/server/services/members-custom-fields/config.ts

@@ -0,0 +1,38 @@
+import {z} from 'zod';
+
+/**
+ * Resolving Ghost's config into the values this service runs on.
+ *
+ * Config arrives untyped: defaults.json supplies the shipped value, and any
+ * higher layer — an operator's config file, an environment variable, argv — can
+ * replace it with anything at all. Turning that into something usable is a
+ * config concern, so it happens here, at the boundary. The service is handed an
+ * answer and never learns where it came from.
+ */
+
+// A ceiling in the form the service needs it. A numeric string is accepted
+// because a value supplied through an environment variable can arrive as one.
+const Ceiling = z
+    .union([z.number(), z.string().trim().min(1)])
+    .transform(Number)
+    .pipe(z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER));
+
+// defaults.json is the single source of the shipped ceiling: it is the lowest
+// config layer and the file an operator actually reads and edits, so it is read
+// rather than restated. Parsed eagerly because a missing or malformed value
+// there is our bug rather than an operator's, and should fail loudly at boot
+// instead of quietly becoming the fallback for everything below.
+const shippedCeiling: number = Ceiling.parse(
+    require('../../../shared/config/defaults.json').members.customFields.maxDefinitions
+);
+
+// A higher layer can override the shipped value with anything. Something
+// unreadable falls back to the shipped ceiling rather than throwing, because an
+// operator can correct this live and a bad value must not take the feature down
+// with it — and rather than meaning "no ceiling", because a typo must not
+// silently remove the safeguard.
+const MaxDefinitions = Ceiling.catch(shippedCeiling);
+
+export function resolveMaxDefinitions(configured: unknown): number {
+    return MaxDefinitions.parse(configured);
+}