ghost 6.64.0 → 6.65.0

package/core/server/services/lib/mailgun-client.js

@@ -3,6 +3,11 @@ const debug = require('@tryghost/debug');
 const logging = require('@tryghost/logging');
 const metrics = require('@tryghost/metrics');
 const errors = require('@tryghost/errors');
+const {
+  addRecipientMessageIds,
+  isTemplatedRecipientMessageId,
+  RECIPIENT_MESSAGE_ID_VARIABLE,
+} = require('./mailgun-recipient-message-id');
 
 const DEFAULT_BATCH_SIZE = 1000;
 
@@ -19,6 +24,7 @@ module.exports = class MailgunClient {
    * Creates the data payload and sends to Mailgun
    *
    * @param {Object} message
+   * @param {boolean} [message.perRecipientMessageId] Give every recipient its own Message-Id header
    * @param {Object} recipientData
    * @param {Array<Object>} replacements
    *
@@ -118,6 +124,14 @@ module.exports = class MailgunClient {
       // Use overriden domain if specified in message
       const mailDomain = message.domainOverride ? message.domainOverride : mailgunConfig.domain;
 
+      if (message.perRecipientMessageId === true) {
+        // one Message-Id per recipient, referenced from a recipient variable like List-Unsubscribe above
+        messageData['recipient-variables'] = JSON.stringify(
+          addRecipientMessageIds(recipientData, { emailId: message.id, domain: mailDomain }),
+        );
+        messageData['h:Message-Id'] = `<%recipient.${RECIPIENT_MESSAGE_ID_VARIABLE}%>`;
+      }
+
       const response = await mailgunInstance.messages.create(mailDomain, messageData);
       metrics.metric('mailgun-send-mail', {
         value: Date.now() - startTime,
@@ -125,7 +139,8 @@ module.exports = class MailgunClient {
       });
 
       return {
-        id: response.id,
+        // a batch of per-recipient Message-Ids has no single provider id
+        id: isTemplatedRecipientMessageId(response.id) ? null : response.id,
       };
     } catch (error) {
       logging.error(error);

package/core/server/services/lib/mailgun-recipient-message-id.js

@@ -0,0 +1,49 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RECIPIENT_MESSAGE_ID_VARIABLE = void 0;
+exports.buildRecipientMessageId = buildRecipientMessageId;
+exports.isTemplatedRecipientMessageId = isTemplatedRecipientMessageId;
+exports.addRecipientMessageIds = addRecipientMessageIds;
+/**
+ * Per-recipient Message-Id values for Mailgun batch sends. Mailgun mints one Message-Id per
+ * API call, so without these every recipient shares it and readers' replies thread together.
+ * Not to be confused with `./mailgun-message-id`, which normalizes the id Mailgun returns.
+ */
+const node_crypto_1 = __importDefault(require("node:crypto"));
+exports.RECIPIENT_MESSAGE_ID_VARIABLE = 'message_id';
+/**
+ * Deterministic per (email, recipient) so a resubmitted batch reuses the same ids; the address
+ * is hashed rather than exposed. Test emails have no email id and get a random prefix instead.
+ * No angle brackets: the header template adds them.
+ */
+function buildRecipientMessageId({ emailId, recipientEmail, domain, }) {
+    const prefix = emailId || node_crypto_1.default.randomUUID();
+    const digest = node_crypto_1.default
+        .createHash('sha256')
+        .update(`${prefix}:${recipientEmail}`)
+        .digest('hex')
+        .slice(0, 32);
+    return `${prefix}.${digest}@${domain}`;
+}
+/**
+ * Mailgun returns the Message-Id it was given, so when the header is a per-recipient template it
+ * comes back unsubstituted and identifies nothing.
+ */
+function isTemplatedRecipientMessageId(messageId) {
+    return (typeof messageId === 'string' &&
+        messageId.includes(`%recipient.${exports.RECIPIENT_MESSAGE_ID_VARIABLE}%`));
+}
+/** Returns a new recipient-variables map with a `message_id` per recipient; the input is not mutated */
+function addRecipientMessageIds(recipientData, { emailId, domain }) {
+    const result = {};
+    for (const [recipientEmail, variables] of Object.entries(recipientData)) {
+        result[recipientEmail] = {
+            ...variables,
+            [exports.RECIPIENT_MESSAGE_ID_VARIABLE]: buildRecipientMessageId({ emailId, recipientEmail, domain }),
+        };
+    }
+    return result;
+}

package/core/server/services/limits.js

@@ -3,7 +3,7 @@ const config = require('../../shared/config');
 const db = require('../data/db');
 const logging = require('@tryghost/logging');
 const { LimitService } = require('@tryghost/limit-service');
-let limitService = new LimitService();
+const limitService = new LimitService();
 
 const init = () => {
   let helpLink;

package/core/server/services/mail/ghost-mailer.js

@@ -105,7 +105,7 @@ function createMailError({ message, err, ignoreDefaultMessage } = { message: ''
   const defaultErrorMessage = tpl(messages.failedSendingEmailError);
 
   const fullErrorMessage = defaultErrorMessage + message;
-  let statusCode = err && err.name === 'RecipientError' ? 400 : 500;
+  const statusCode = err && err.name === 'RecipientError' ? 400 : 500;
   return new errors.EmailError({
     message: ignoreDefaultMessage ? message : fullErrorMessage,
     err: err,

package/core/server/services/member-attribution/member-attribution-service.js

@@ -34,7 +34,7 @@ class MemberAttributionService {
 
     // We consider only select internal context sources
     if (['import', 'api', 'admin'].includes(source)) {
-      let attribution = {
+      const attribution = {
         id: null,
         type: null,
         url: null,

package/core/server/services/member-attribution/referrer-translator.js

@@ -76,7 +76,7 @@ class ReferrerTranslator {
 
     // In finding the 'content' that got the user to sign up, we want the latest entry with referrer data
     for (const item of history) {
-      let refUrl = this.getUrlFromStr(item.referrerUrl);
+      const refUrl = this.getUrlFromStr(item.referrerUrl);
       if (refUrl?.hostname === 'checkout.stripe.com') {
         // Ignore stripe, because second try payments should not be attributed to Stripe
         continue;

package/core/server/services/members/account-service.js

@@ -71,11 +71,11 @@ class MemberAccountService {
             withRelated: WRITE_RELATIONS,
         });
         if (plannedMetafields) {
-            // A member is recorded as the author of their own answers, and is the one
-            // writer whose changes leave nothing in the staff action log: that log
-            // records what staff did.
+            // A member is recorded as the author of their own answers, made from their
+            // account, which is Portal's.
             await this.#metafieldValues.applyWrite(memberId, plannedMetafields, {
                 writtenBy: { type: 'member', id: memberId },
+                source: 'portal',
             });
         }
         // Read back rather than returning what was written: a member is told what

package/core/server/services/members/api.js

@@ -76,7 +76,7 @@ function createApiInstance(config) {
           if (process.env.NODE_ENV !== 'production') {
             logging.warn(message.text);
           }
-          let msg = Object.assign(
+          const msg = Object.assign(
             {
               from: config.getEmailSupportAddress(),
               subject: 'Signin',

package/core/server/services/members/import-export/import/importer.js

@@ -7,6 +7,7 @@ const moment_timezone_1 = __importDefault(require("moment-timezone"));
 const completion_email_1 = __importDefault(require("./completion-email"));
 const csv_1 = require("../csv");
 const csv_2 = require("@tryghost/metafield-types/csv");
+const members_import_job_1 = __importDefault(require("../../jobs/members-import-job"));
 const metrics = require('@tryghost/metrics');
 const errors = require('@tryghost/errors');
 const logging = require('@tryghost/logging');
@@ -53,10 +54,10 @@ class MembersCSVImporter {
     _metafields;
     _email;
     _report;
-    _addJob;
+    _dispatchJob;
     _getTimezone;
     _getInlineThreshold;
-    constructor({ knex, readRows, spool, members, tiers, stripe, gifts, metafields, email, report, addJob, getTimezone, getInlineThreshold, }) {
+    constructor({ knex, readRows, spool, members, tiers, stripe, gifts, metafields, email, report, dispatchJob, getTimezone, getInlineThreshold, }) {
         this._knex = knex;
         this._readRows = readRows;
         this._spool = spool;
@@ -67,7 +68,7 @@ class MembersCSVImporter {
         this._metafields = metafields;
         this._email = email;
         this._report = report;
-        this._addJob = addJob;
+        this._dispatchJob = dispatchJob;
         this._getTimezone = getTimezone;
         this._getInlineThreshold = getInlineThreshold;
     }
@@ -79,35 +80,43 @@ class MembersCSVImporter {
             const result = await this.importRows(rows, labelName, extraLabels, verificationTrigger);
             return { deferred: false, originalImportSize: rows.length, result };
         }
-        await this.deferImport(rows, { labelName, extraLabels, requestUserEmail: request.requestUserEmail }, verificationTrigger);
+        await this.deferImport(rows, {
+            labelName,
+            extraLabels,
+            requestUserEmail: request.requestUserEmail,
+        });
         return { deferred: true, originalImportSize: rows.length };
     }
     async importInline(request, verificationTrigger) {
         const rows = await this._readRows(request.filePath, request.mapping);
         return this.importRows(rows, buildImportLabelName(this._getTimezone()), request.extraLabels ?? [], verificationTrigger);
     }
-    async deferImport(rows, { labelName, extraLabels, requestUserEmail, }, verificationTrigger) {
+    async deferImport(rows, { labelName, extraLabels, requestUserEmail, }) {
         // Resolved here, not at the API boundary, so the owner lookup only runs when
         // a request without a user actually reaches the deferred path.
         const emailRecipient = requestUserEmail ?? (await this._email.getDefaultRecipient());
-        const spooled = await this._spool.write(rows);
+        const spoolKey = await this._spool.write(rows);
         logging.info({ event: { name: 'members.import.queued' }, rows: rows.length }, 'Members import queued');
-        this._addJob({
-            job: () => this.runImportJob(spooled, { labelName, extraLabels, emailRecipient }, verificationTrigger),
-            offloaded: false,
-            name: 'members-import',
-        });
+        try {
+            await this._dispatchJob(new members_import_job_1.default({ spoolKey, labelName, extraLabels, emailRecipient }));
+        }
+        catch (error) {
+            // No job will ever read the rows, so they are not left behind for one.
+            await this.settle(() => this._spool.remove(spoolKey));
+            throw error;
+        }
     }
-    // Must resolve in every case: the job manager reads a rejected inline job as a defect
-    // in the job itself, and there is no retry behind it.
-    async runImportJob(spooled, { labelName, extraLabels, emailRecipient, }, verificationTrigger) {
+    // The members-import job handler. Resolves in every case, as the legacy inline job did:
+    // everything that can fail is already reported here, so a rejection would only make the
+    // jobs service report it again, and there is no retry behind it.
+    async handle({ spoolKey, labelName, extraLabels, emailRecipient }, verificationTrigger) {
         const startedAt = Date.now();
         logging.info({ event: { name: 'members.import.started' } }, 'Members import started');
         // Null until the import produces one: parsing and mapping already happened inside
         // the request, so anything failing from here is ours rather than the file's.
         let result = null;
         try {
-            const spooledRows = await spooled.read();
+            const spooledRows = await this._spool.read(spoolKey);
             result = await this.importRows(spooledRows, labelName, extraLabels, verificationTrigger);
         }
         catch (error) {
@@ -115,7 +124,7 @@ class MembersCSVImporter {
             this._report(error);
         }
         finally {
-            await this.settle(() => spooled.remove());
+            await this.settle(() => this._spool.remove(spoolKey));
         }
         // Whatever became of it, the publisher hears exactly once. If this is what fails,
         // there is nobody left to tell but us.

package/core/server/services/members/import-export/import/spool.js

@@ -4,27 +4,23 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createRowSpool = createRowSpool;
-const node_os_1 = __importDefault(require("node:os"));
-const node_path_1 = __importDefault(require("node:path"));
 const node_crypto_1 = __importDefault(require("node:crypto"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-// Spools import rows to a private JSON file under the OS temp dir, so a deferred
+// Spools import rows to a JSON file in the imports storage adapter, so a deferred
 // import can hand them to a background job and read them back after the request has
 // already returned. The rows go in and come out as MemberImportRow, so nothing but
 // the import's own row shape crosses this boundary.
-function createRowSpool() {
+function createRowSpool(storage) {
     return {
         async write(rows) {
-            const spoolPath = node_path_1.default.join(node_os_1.default.tmpdir(), `members-import-${node_crypto_1.default.randomUUID()}.json`);
-            await fs_extra_1.default.writeFile(spoolPath, JSON.stringify(rows), { mode: 0o600 });
-            return {
-                async read() {
-                    return JSON.parse(await fs_extra_1.default.readFile(spoolPath, 'utf8'));
-                },
-                async remove() {
-                    await fs_extra_1.default.remove(spoolPath);
-                },
-            };
+            const key = `members-import-${node_crypto_1.default.randomUUID()}.json`;
+            await storage.saveRaw(Buffer.from(JSON.stringify(rows)), key);
+            return key;
+        },
+        async read(key) {
+            return JSON.parse((await storage.read({ path: key })).toString('utf8'));
+        },
+        async remove(key) {
+            await storage.delete(key);
         },
     };
 }

package/core/server/services/members/import-export/index.js

@@ -64,11 +64,13 @@ function makeImporter(deps) {
         // An import has no id to give until runs are tracked, so it names its kind only.
         applyWrite: (memberId, plan, executor) => deps.metafields.values.applyWrite(memberId, plan, {
             writtenBy: { type: 'import', id: null },
+            source: 'import',
             executor,
         }),
     };
-    // Inline jobs never reach the job manager's Sentry handler, which is wired to the
-    // offloaded worker path only, so a throw here would be seen by nobody.
+    // The import job never rejects, so the jobs service never sees its failures, and the
+    // inline path's bookkeeping failures never reach the request: a throw here would be
+    // seen by nobody.
     const report = (error) => {
         try {
             logging.error({ event: { name: 'members.import.error' }, err: error }, '[Background Job] members-import error');
@@ -81,7 +83,7 @@ function makeImporter(deps) {
     return new importer_1.default({
         knex: deps.knex,
         readRows: reader_1.default,
-        spool: (0, spool_1.createRowSpool)(),
+        spool: (0, spool_1.createRowSpool)(require('../../adapter-manager').default.getAdapter('storage:imports')),
         members,
         tiers: {
             getDefault: deps.getDefaultTier,
@@ -95,7 +97,7 @@ function makeImporter(deps) {
         metafields,
         email,
         report,
-        addJob: deps.addJob,
+        dispatchJob: deps.dispatchJob,
         getTimezone: deps.getTimezone,
         getInlineThreshold: deps.getInlineThreshold,
     });

package/core/server/services/members/jobs/index.js

@@ -5,7 +5,7 @@ const CleanExpiredCompedJob = require('./clean-expired-comped-job').default;
 const cleanTokensTask = require('./clean-tokens-task').default;
 const cleanExpiredCompedTask = require('./clean-expired-comped-task').default;
 
-let hasScheduled = {
+const hasScheduled = {
   expiredComped: false,
   tokens: false,
 };

package/core/server/services/members/jobs/members-import-job.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const job_1 = require("../../jobs-service/job");
+class MembersImportJob extends job_1.Job {
+    static type = 'members-import';
+    spoolKey;
+    labelName;
+    extraLabels;
+    emailRecipient;
+    constructor(data) {
+        super();
+        this.spoolKey = data.spoolKey;
+        this.labelName = data.labelName;
+        this.extraLabels = data.extraLabels;
+        this.emailRecipient = data.emailRecipient;
+    }
+}
+exports.default = MembersImportJob;

package/core/server/services/members/member-avatar.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.memberAvatarImage = memberAvatarImage;
+const config = require('../../../shared/config');
+/**
+ * The avatar shown for a member: their Gravatar, served blank when they have none, unless
+ * the site has turned Gravatar off. Nothing is stored, so every place that shows a member
+ * works it out from their email the same way.
+ */
+function memberAvatarImage(email) {
+    if (!email || config.isPrivacyDisabled('useGravatar')) {
+        return null;
+    }
+    // Required on first use, as the member model did. The image library is untyped JS, so
+    // `gravatar` is not checked here.
+    const { gravatar } = require('../../lib/image');
+    return gravatar.url(email, { size: 250, default: 'blank' });
+}

package/core/server/services/members/members-api/controllers/member-controller.js

@@ -57,7 +57,7 @@ module.exports = class MemberController {
       });
     }
 
-    let tokenData = {};
+    const tokenData = {};
     try {
       const member = await this._memberRepository.getByToken(identity);
       tokenData.oldEmail = member.get('email');
@@ -88,7 +88,7 @@ module.exports = class MemberController {
       const cancelAtPeriodEnd = req.body.cancel_at_period_end;
       const smartCancel = req.body.smart_cancel;
       const cancellationReason = req.body.cancellation_reason;
-      let ghostPriceId = req.body.priceId;
+      const ghostPriceId = req.body.priceId;
       const tierId = req.body.tierId;
       const cadence = req.body.cadence;
 

package/core/server/services/members/members-api/members-api.js

@@ -7,7 +7,6 @@ const PaymentsService = require('./services/payments-service');
 const TokenService = require('./services/token-service');
 const GeolocationService = require('./services/geolocation-service');
 const MemberBREADService = require('./services/member-bread-service');
-const metafields = require('../../members-metafields');
 const { MemberAccountService } = require('../account-service');
 const MemberRepository = require('./repositories/member-repository');
 const NextPaymentCalculator = require('./services/next-payment-calculator');
@@ -128,6 +127,7 @@ module.exports = function MembersAPI({
     labsService,
     memberAttributionService,
     MemberEmailChangeEvent,
+    metafieldValues,
     AutomatedEmailRecipient,
     giftSubscriptions: giftService,
   });
@@ -285,7 +285,7 @@ module.exports = function MembersAPI({
       return null;
     }
 
-    let member = oldEmail
+    const member = oldEmail
       ? await getMemberIdentityData(oldEmail)
       : await getMemberIdentityData(email);
 
@@ -365,7 +365,7 @@ module.exports = function MembersAPI({
     memberBREADService,
     members: users,
     emailSuppressionList,
-    metafieldValues: metafields.values,
+    metafieldValues,
   });
 
   async function getMemberIdentity(transientId) {
@@ -434,7 +434,7 @@ module.exports = function MembersAPI({
     }
 
     // max request time is 500ms so shouldn't slow requests down too much
-    let geolocation = JSON.stringify(await geolocationService.getGeolocationFromIP(ip));
+    const geolocation = JSON.stringify(await geolocationService.getGeolocationFromIP(ip));
     if (geolocation) {
       await users.update({ geolocation }, { id: member.id });
     }

package/core/server/services/members/members-api/repositories/event-repository.js

@@ -12,6 +12,7 @@ const {
 } = require('@tryghost/mongo-utils');
 const { default: ObjectID } = require('bson-objectid');
 const db = require('../../../../data/db');
+const { memberAvatarImage } = require('../../member-avatar');
 
 /**
  * This mongo transformer ignores the provided filter option and replaces the filter with a custom filter that was provided to the transformer. Allowing us to set a mongo filter instead of a string based NQL filter.
@@ -42,6 +43,7 @@ module.exports = class EventRepository {
     memberAttributionService,
     urlService,
     MemberEmailChangeEvent,
+    metafieldValues,
     AutomatedEmailRecipient,
     giftSubscriptions,
   }) {
@@ -60,6 +62,7 @@ module.exports = class EventRepository {
     this._EmailSpamComplaintEvent = EmailSpamComplaintEvent;
     this._memberAttributionService = memberAttributionService;
     this._MemberEmailChangeEvent = MemberEmailChangeEvent;
+    this._metafieldValues = metafieldValues;
     this._AutomatedEmailRecipient = AutomatedEmailRecipient;
     this._giftSubscriptions = giftSubscriptions;
     this._knex = db.knex;
@@ -99,6 +102,7 @@ module.exports = class EventRepository {
         { type: 'login_event', action: 'getLoginEvents' },
         { type: 'payment_event', action: 'getPaymentEvents' },
         { type: 'email_change_event', action: 'getEmailChangeEvent' },
+        { type: 'metafield_change_event', action: 'getMetafieldChangeEvents' },
         { type: 'gift_purchase_event', action: 'getGiftPurchaseEvents' },
         { type: 'gift_redemption_event', action: 'getGiftRedemptionEvents' },
         { type: 'gift_ended_event', action: 'getGiftEndedEvents' },
@@ -652,7 +656,7 @@ module.exports = class EventRepository {
 
     filter = this.removePostIdFilter(otherFilter); //Remove post_id filter as we don't need it in the query
 
-    let postClicksQuery = postId
+    const postClicksQuery = postId
       ? `SELECT
                     mce.id,
                     mce.member_id,
@@ -1012,6 +1016,37 @@ module.exports = class EventRepository {
     };
   }
 
+  async getMetafieldChangeEvents(options = {}, filter) {
+    // The entries belong to the metafields domain, which reads them without a model; the
+    // feed maps its own filter names onto the table's columns and adds each member's avatar.
+    const columnFilter = filter
+      ? chainTransformers(
+          ...mapKeys({
+            'data.created_at': 'created_at',
+            'data.member_id': 'member_id',
+          }),
+        )(filter)
+      : undefined;
+
+    const { events, total } = await this._metafieldValues.browseChangeEvents({
+      // The feed's limit arrives as the request sent it, which can be a numeric string
+      // or `all`; the service takes a number, or nothing for every entry.
+      limit: options.limit === 'all' ? undefined : Number(options.limit),
+      filter: columnFilter,
+    });
+
+    return {
+      data: events.map((event) => ({
+        type: 'metafield_change_event',
+        data: {
+          ...event,
+          member: { ...event.member, avatar_image: memberAvatarImage(event.member.email) },
+        },
+      })),
+      meta: { pagination: { total } },
+    };
+  }
+
   async getAutomatedEmailSentEvents(options = {}, filter) {
     options = {
       ...options,