ghost 6.57.1 → 6.58.0

package/core/server/services/email-address/email-address-service.ts

@@ -1,241 +0,0 @@
-import logging from '@tryghost/logging';
-import EmailAddressParser, {EmailAddress} from './email-address-parser.js';
-
-export type EmailAddresses = {
-    from: EmailAddress,
-    replyTo?: EmailAddress
-}
-
-export type EmailAddressesValidation = {
-    allowed: boolean,
-    verificationEmailRequired: boolean,
-    reason?: string
-}
-
-export type EmailAddressType = 'from' | 'replyTo';
-
-type GetAddressOptions = {
-    useFallbackAddress: boolean
-}
-
-export class EmailAddressService {
-    #getManagedEmailEnabled: () => boolean;
-    #getSendingDomain: () => string | null;
-    #getDefaultEmail: () => EmailAddress;
-    #getFallbackDomain: () => string | null;
-    #getFallbackEmail: () => EmailAddress | null;
-    #getMembersSupportAddress: () => string;
-    #isValidEmailAddress: (email: string) => boolean;
-
-    constructor(dependencies: {
-        getManagedEmailEnabled: () => boolean,
-        getSendingDomain: () => string | null,
-        getFallbackDomain: () => string | null,
-        getDefaultEmail: () => EmailAddress,
-        getFallbackEmail: () => string | null,
-        getMembersSupportAddress: () => string,
-        isValidEmailAddress: (email: string) => boolean
-    }) {
-        this.#getManagedEmailEnabled = dependencies.getManagedEmailEnabled;
-        this.#getSendingDomain = dependencies.getSendingDomain;
-        this.#getFallbackDomain = dependencies.getFallbackDomain;
-        this.#getDefaultEmail = dependencies.getDefaultEmail;
-        this.#getMembersSupportAddress = dependencies.getMembersSupportAddress;
-        this.#getFallbackEmail = () => {
-            const fallbackAddress = dependencies.getFallbackEmail();
-            if (!fallbackAddress) {
-                return null;
-            }
-            return EmailAddressParser.parse(fallbackAddress);
-        };
-        this.#isValidEmailAddress = dependencies.isValidEmailAddress;
-    }
-
-    get sendingDomain(): string | null {
-        return this.#getSendingDomain();
-    }
-
-    get fallbackDomain(): string | null {
-        return this.#getFallbackDomain();
-    }
-
-    get managedEmailEnabled(): boolean {
-        return this.#getManagedEmailEnabled();
-    }
-
-    get defaultFromEmail(): EmailAddress {
-        return this.#getDefaultEmail();
-    }
-
-    get fallbackEmail(): EmailAddress | null {
-        return this.#getFallbackEmail();
-    }
-
-    /**
-     * Get the actual sender address for member emails after DMARC transformation.
-     * On managed platforms, this may differ from the configured support address to ensure DMARC compliance.
-     */
-    getMembersSupportAddress(): string {
-        const configuredAddress = this.#getMembersSupportAddress();
-        const transformed = this.getAddressFromString(configuredAddress);
-        return transformed.from.address;
-    }
-
-    getAddressFromString(from: string, replyTo?: string): EmailAddresses {
-        const parsedFrom = EmailAddressParser.parse(from);
-        const parsedReplyTo = replyTo ? EmailAddressParser.parse(replyTo) : undefined;
-
-        return this.getAddress({
-            from: parsedFrom ?? this.defaultFromEmail,
-            replyTo: parsedReplyTo ?? undefined
-        });
-    }
-
-    /**
-     * When sending an email, we should always ensure DMARC alignment.
-     * Because of that, we restrict which email addresses we send from. All emails should be either
-     * send from a configured domain (hostSettings.managedEmail.sendingDomains), or from the configured email address (mail.from).
-     *
-     * If we send an email from an email address that doesn't pass, we'll just default to the default email address,
-     * and instead add a replyTo email address from the requested from address.
-     */
-    getAddress(preferred: EmailAddresses, options: GetAddressOptions = {useFallbackAddress: false}): EmailAddresses {
-        if (preferred.replyTo && !this.#isValidEmailAddress(preferred.replyTo.address)) {
-            // The default from-address itself is allowed to fail generic email validation
-            // (e.g. noreply@127.0.0.1 on an IP-only install — see the same carve-out in
-            // validate() below), so don't error-log when the replyTo just happens to match
-            // it (case-insensitively — members_support_address is user-entered, so a case
-            // variant of the same address shouldn't defeat the carve-out). This also
-            // silences a genuinely malformed mail:from, not just the known IP-domain case —
-            // demote to warn instead of dropping the signal entirely.
-            if (preferred.replyTo.address.toLowerCase() !== this.defaultFromEmail.address.toLowerCase()) {
-                logging.error(`[EmailAddresses] Invalid replyTo address: ${preferred.replyTo.address}`);
-            } else {
-                logging.warn(`[EmailAddresses] replyTo address ${preferred.replyTo.address} matches the default from-address and failed generic validation; removing it`);
-            }
-            // Remove invalid replyTo addresses
-            preferred.replyTo = undefined;
-        }
-
-        // Validate the from address
-        if (!this.#isValidEmailAddress(preferred.from.address)) {
-            // Never allow an invalid email address
-            return {
-                from: this.defaultFromEmail,
-                replyTo: preferred.replyTo || undefined
-            };
-        }
-
-        if (!this.managedEmailEnabled) {
-            // Self hoster or legacy Ghost Pro
-            return preferred;
-        }
-
-        // Case: use fallback address when warming up custom domain
-        if (options.useFallbackAddress) {
-            const fallbackEmail = this.fallbackEmail;
-            if (fallbackEmail) {
-                if (!fallbackEmail.name) {
-                    fallbackEmail.name = preferred.from.name || this.defaultFromEmail.name;
-                }
-
-                return {
-                    from: fallbackEmail,
-                    replyTo: preferred.replyTo || preferred.from || this.defaultFromEmail
-                };
-            } else {
-                logging.error('[EmailAddresses] Fallback email address is not configured, cannot use fallback address for sending email.');
-            }
-        }
-
-        // Case: always allow the default from address
-        if (preferred.from.address === this.defaultFromEmail.address) {
-            if (!preferred.from.name) {
-                // Use the default sender name if it is missing
-                preferred.from.name = this.defaultFromEmail.name;
-            }
-
-            return preferred;
-        }
-
-        if (this.sendingDomain) {
-            // Check if FROM address is from the sending domain
-            if (preferred.from.address.endsWith(`@${this.sendingDomain}`)) {
-                return preferred;
-            }
-
-            // Not an error: this is the expected, handled fallback for any sender address that
-            // doesn't match the sending domain (staff/member/newsletter senders routinely don't) —
-            // we fall through to the default from-address below. Warn-level, not error-level.
-            logging.warn(`[EmailAddresses] Cannot send emails from ${preferred.from.address} when sending domain is ${this.sendingDomain} — falling back to the default from-address`);
-        }
-
-        // Only allow to send from the configured from address
-        const address = {
-            from: this.defaultFromEmail,
-            replyTo: preferred.replyTo || preferred.from
-        };
-
-        // Do allow to change the sender name if requested
-        if (preferred.from.name) {
-            address.from.name = preferred.from.name;
-        }
-
-        if (address.replyTo.address === address.from.address) {
-            return {
-                from: address.from
-            };
-        }
-
-        return address;
-    }
-
-    /**
-     * When changing any from or reply to addresses in the system, we need to validate them
-     */
-    validate(email: string, type: EmailAddressType): EmailAddressesValidation {
-        if (!this.#isValidEmailAddress(email)) {
-            // Never allow an invalid email address
-            return {
-                allowed: email === this.defaultFromEmail.address, // Localhost email noreply@127.0.0.1 is marked as invalid, but we should allow it
-                verificationEmailRequired: false,
-                reason: 'invalid'
-            };
-        }
-
-        if (!this.managedEmailEnabled) {
-            // Self hoster or legacy Ghost Pro
-            return {
-                allowed: true,
-                verificationEmailRequired: false // Self hosters don't need to verify email addresses
-            };
-        }
-
-        if (this.sendingDomain) {
-            // Only allow it if it ends with the sending domain
-            if (email.endsWith(`@${this.sendingDomain}`)) {
-                return {
-                    allowed: true,
-                    verificationEmailRequired: false
-                };
-            }
-
-            // Use same restrictions as one without a sending domain for other addresses
-        }
-
-        // Only allow to edit the replyTo address, with verification
-        if (type === 'replyTo') {
-            return {
-                allowed: true,
-                verificationEmailRequired: email !== this.defaultFromEmail.address
-            };
-        }
-
-        // Not allowed to change from
-        return {
-            allowed: email === this.defaultFromEmail.address,
-            verificationEmailRequired: false,
-            reason: 'not allowed'
-        };
-    }
-}

package/core/server/services/email-analytics/automation-email-analytics-batch-processor.ts

@@ -1,130 +0,0 @@
-import {normalizeMailgunMessageId} from '../automations/mailgun-message-id';
-import type * as automationsApi from '../automations/automations-api';
-import type {AutomatedEmailEvents, AutomatedEmailRecipientWithMailgunId} from '../automations/automations-repository';
-import type {BatchEventProcessor} from './batch-event-processor';
-import {EventProcessingResult} from './event-processing-result';
-
-type AutomationsApi = {
-    getAutomatedEmailRecipientsByMailgunIds: typeof automationsApi.getAutomatedEmailRecipientsByMailgunIds;
-    trackEmailDeliveredAndOpened: typeof automationsApi.trackEmailDeliveredAndOpened;
-};
-
-type EmailAnalyticsEvent = {
-    type: string;
-    providerId: string;
-    timestamp: Date;
-};
-
-const getMailgunMessageIds = (events: Iterable<EmailAnalyticsEvent>): Set<string> => {
-    const result = new Set<string>();
-    for (const {providerId} of events) {
-        result.add(normalizeMailgunMessageId(providerId));
-    }
-    return result;
-};
-
-const getAutomatedEmailRecipients = async (
-    automationsApi: AutomationsApi,
-    mailgunMessageIds: Set<string>
-): Promise<AutomatedEmailRecipientWithMailgunId[]> => (
-    mailgunMessageIds.size
-        ? await automationsApi.getAutomatedEmailRecipientsByMailgunIds(Array.from(mailgunMessageIds))
-        : []
-);
-
-const getAutomatedEmailRecipientsByMessageId = (
-    automatedEmailRecipients: Iterable<AutomatedEmailRecipientWithMailgunId>
-): Map<string, AutomatedEmailRecipientWithMailgunId> => {
-    const result = new Map<string, AutomatedEmailRecipientWithMailgunId>();
-    for (const automatedEmailRecipient of automatedEmailRecipients) {
-        result.set(automatedEmailRecipient.mailgun_message_id, automatedEmailRecipient);
-    }
-    return result;
-};
-
-const trackEarliest = (
-    map: Map<string, AutomatedEmailEvents>,
-    recipient: AutomatedEmailRecipientWithMailgunId,
-    field: 'deliveredAt' | 'openedAt',
-    timestamp: Date
-): void => {
-    const events = map.get(recipient.id) ?? {
-        automationActionRevisionId: recipient.automation_action_revision_id
-    };
-    const existing = events[field];
-    if (!existing || timestamp < existing) {
-        events[field] = timestamp;
-    }
-    map.set(recipient.id, events);
-};
-
-export class AutomationEmailAnalyticsBatchProcessor implements BatchEventProcessor {
-    #automationsApi;
-
-    constructor({
-        automationsApi
-    }: {
-        automationsApi: AutomationsApi
-    }) {
-        this.#automationsApi = automationsApi;
-    }
-
-    async processBatch(
-        events: ReadonlyArray<EmailAnalyticsEvent>,
-        result: EventProcessingResult,
-        fetchData: {lastEventTimestamp?: Date}
-    ): Promise<void> {
-        const mailgunMessageIds = getMailgunMessageIds(events);
-
-        const automatedEmailRecipients = await getAutomatedEmailRecipients(
-            this.#automationsApi,
-            mailgunMessageIds
-        );
-        const automatedEmailRecipientsByMessageId = getAutomatedEmailRecipientsByMessageId(automatedEmailRecipients);
-
-        const eventsByAutomatedEmailRecipientId = new Map<string, AutomatedEmailEvents>();
-
-        for (const event of events) {
-            if (!fetchData.lastEventTimestamp || event.timestamp > fetchData.lastEventTimestamp) {
-                fetchData.lastEventTimestamp = event.timestamp;
-            }
-
-            let eventResult: EventProcessingResult;
-
-            const getRecipient = () => {
-                const mailgunMessageId = normalizeMailgunMessageId(event.providerId);
-                return automatedEmailRecipientsByMessageId.get(mailgunMessageId);
-            };
-
-            switch (event.type) {
-            case 'delivered': {
-                const recipient = getRecipient();
-                if (recipient) {
-                    trackEarliest(eventsByAutomatedEmailRecipientId, recipient, 'deliveredAt', event.timestamp);
-                    eventResult = new EventProcessingResult({delivered: 1});
-                } else {
-                    eventResult = new EventProcessingResult({unprocessable: 1});
-                }
-                break;
-            }
-            case 'opened': {
-                const recipient = getRecipient();
-                if (recipient) {
-                    trackEarliest(eventsByAutomatedEmailRecipientId, recipient, 'openedAt', event.timestamp);
-                    eventResult = new EventProcessingResult({opened: 1});
-                } else {
-                    eventResult = new EventProcessingResult({unprocessable: 1});
-                }
-                break;
-            }
-            default:
-                eventResult = new EventProcessingResult({unhandled: 1});
-                break;
-            }
-
-            result.merge(eventResult);
-        }
-
-        await this.#automationsApi.trackEmailDeliveredAndOpened(eventsByAutomatedEmailRecipientId);
-    }
-}

package/core/server/services/email-analytics/batch-event-processor.ts

@@ -1,18 +0,0 @@
-import type {EventProcessingResult} from './event-processing-result';
-
-export type BatchEventProcessor = {
-    processBatch(
-        events: any[],
-        result: EventProcessingResult,
-        fetchData: {lastEventTimestamp?: Date}
-    ): Promise<void>;
-
-    aggregate?: (options: {
-        includeOpenedEvents: boolean;
-        processingResult: EventProcessingResult;
-        isFinal: boolean;
-    }) => Promise<null | {
-        emailAggregationTimeMs: number;
-        memberAggregationTimeMs: number;
-    }>;
-};

package/core/server/services/email-analytics/email-analytics-provider-mailgun.js

@@ -1,57 +0,0 @@
-const MailgunClient = require('../lib/mailgun-client');
-
-const DEFAULT_EVENT_FILTER = 'delivered OR opened OR failed OR unsubscribed OR complained';
-const PAGE_LIMIT = 300;
-
-class EmailAnalyticsProviderMailgun {
-    mailgunClient;
-
-    constructor({config, settings, tags}) {
-        this.mailgunClient = new MailgunClient({config, settings});
-        this.tags = tags;
-    }
-
-    /**
-     * Fetch from the last known timestamp-TRUST_THRESHOLD then work forwards
-     * through pages until we get a blank response. This lets us get events
-     * quicker than the TRUST_THRESHOLD
-     *
-     * @param {Function} batchHandler
-     * @param {Object} [options]
-     * @param {number} [options.maxEvents] Not a strict maximum. We stop fetching after we reached the maximum AND received at least one event after begin (not equal) to prevent deadlocks.
-     * @param {Date} [options.begin]
-     * @param {Date} [options.end]
-     * @param {String[]} [options.events]
-     */
-    fetchLatest(batchHandler, options) {
-        const mailgunOptions = {
-            limit: PAGE_LIMIT,
-            event: options?.events ? options.events.join(' OR ') : DEFAULT_EVENT_FILTER,
-            tags: this.tags.join(' AND '),
-            begin: options.begin ? options.begin.getTime() / 1000 : undefined,
-            end: options.end ? options.end.getTime() / 1000 : undefined,
-            ascending: 'yes'
-        };
-
-        return this.#fetchAnalytics(mailgunOptions, batchHandler, {
-            maxEvents: options.maxEvents
-        });
-    }
-
-    /**
-     * @param {Object} mailgunOptions
-     * @param {number} mailgunOptions.limit
-     * @param {string} mailgunOptions.event
-     * @param {string} mailgunOptions.tags
-     * @param {string} [mailgunOptions.begin]
-     * @param {string} [mailgunOptions.ascending]
-     * @param {Function} batchHandler
-     * @param {Object} [options]
-     * @param {number} [options.maxEvents] Not a strict maximum. We stop fetching after we reached the maximum AND received at least one event after begin (not equal) to prevent deadlocks.
-     */
-    async #fetchAnalytics(mailgunOptions, batchHandler, options) {
-        return await this.mailgunClient.fetchEvents(mailgunOptions, batchHandler, options);
-    }
-}
-
-module.exports = EmailAnalyticsProviderMailgun;

package/core/server/services/email-analytics/event-processing-result.ts

@@ -1,66 +0,0 @@
-type EventProcessingResultInput = Partial<Omit<EventProcessingResult, 'totalEvents' | 'merge'>>;
-
-export class EventProcessingResult {
-    // counts
-    delivered: number = 0;
-    opened: number = 0;
-    temporaryFailed: number = 0;
-    permanentFailed: number = 0;
-    unsubscribed: number = 0;
-    complained: number = 0;
-    unhandled: number = 0;
-    unprocessable: number = 0;
-
-    // processing failures are counted separately in addition to event type counts
-    processingFailures: number = 0;
-
-    // ids seen whilst processing ready for passing to stats aggregator
-    emailIds: string[] = [];
-    memberIds: string[] = [];
-
-    constructor(result: EventProcessingResultInput = {}) {
-        this.merge(result);
-    }
-
-    reset(): void {
-        this.delivered = 0;
-        this.opened = 0;
-        this.temporaryFailed = 0;
-        this.permanentFailed = 0;
-        this.unsubscribed = 0;
-        this.complained = 0;
-        this.unhandled = 0;
-        this.unprocessable = 0;
-        this.processingFailures = 0;
-        this.emailIds = [];
-        this.memberIds = [];
-    }
-
-    get totalEvents(): number {
-        return this.delivered
-            + this.opened
-            + this.temporaryFailed
-            + this.permanentFailed
-            + this.unsubscribed
-            + this.complained
-            + this.unhandled
-            + this.unprocessable;
-    }
-
-    merge(other: EventProcessingResultInput = {}): void {
-        this.delivered += other.delivered || 0;
-        this.opened += other.opened || 0;
-        this.temporaryFailed += other.temporaryFailed || 0;
-        this.permanentFailed += other.permanentFailed || 0;
-        this.unsubscribed += other.unsubscribed || 0;
-        this.complained += other.complained || 0;
-        this.unhandled += other.unhandled || 0;
-        this.unprocessable += other.unprocessable || 0;
-
-        this.processingFailures += other.processingFailures || 0;
-
-        // TODO: come up with a cleaner way to merge these without churning through Array and Set
-        this.emailIds = Array.from(new Set([...this.emailIds, ...(other.emailIds || [])])).filter(Boolean);
-        this.memberIds = Array.from(new Set([...this.memberIds, ...(other.memberIds || [])])).filter(Boolean);
-    }
-}

package/core/server/services/email-analytics/events/start-automation-email-analytics-job-event.ts

@@ -1,14 +0,0 @@
-/**
- * This event lets the job manager ask the main thread to start the automation email analytics pipeline.
- */
-export class StartAutomationEmailAnalyticsJobEvent {
-    readonly timestamp: Date;
-
-    constructor(timestamp: Date) {
-        this.timestamp = timestamp;
-    }
-
-    static create(timestamp = new Date()) {
-        return new StartAutomationEmailAnalyticsJobEvent(timestamp);
-    }
-};

package/core/server/services/email-analytics/events/start-email-analytics-job-event.ts

@@ -1,15 +0,0 @@
-/**
- * This is an event that is used to circumvent the job manager that currently isn't able to run scheduled jobs on the main thread (not offloaded).
- * We simply emit this event in the job manager and listen for it on the main thread.
- */
-export class StartEmailAnalyticsJobEvent {
-    readonly timestamp: Date;
-
-    constructor(timestamp: Date) {
-        this.timestamp = timestamp;
-    }
-
-    static create(timestamp = new Date()) {
-        return new StartEmailAnalyticsJobEvent(timestamp);
-    }
-};

package/core/server/services/email-analytics/index.ts

@@ -1,109 +0,0 @@
-// @ts-expect-error This module lacks type definitions.
-import EmailAnalyticsServiceWrapper from './email-analytics-service-wrapper';
-import config from '../../../shared/config';
-// @ts-expect-error This module lacks type definitions.
-import {NewsletterEmailAnalyticsBatchProcessor} from './newsletter-email-analytics-batch-processor';
-// @ts-expect-error This module lacks type definitions.
-import NewsletterEmailEventStorage from '../email-service/newsletter-email-event-storage';
-// @ts-expect-error This module lacks type definitions.
-import EmailEventProcessor from '../email-service/email-event-processor';
-// @ts-expect-error This module lacks type definitions.
-import db from '../../data/db';
-import membersService from '../members';
-// @ts-expect-error This module lacks type definitions.
-import emailSuppressionList from '../email-suppression-list';
-// @ts-expect-error This module lacks type definitions.
-import {EmailRecipientFailure, EmailSpamComplaintEvent, Email} from '../../models';
-// @ts-expect-error This module lacks type definitions.
-import domainEvents from '@tryghost/domain-events';
-// @ts-expect-error This module lacks type definitions.
-import prometheusClient from '../../../shared/prometheus-client';
-import {queries} from './lib/queries';
-import {StartEmailAnalyticsJobEvent} from './events/start-email-analytics-job-event';
-import {StartAutomationEmailAnalyticsJobEvent} from './events/start-automation-email-analytics-job-event';
-import {AUTOMATION_EMAIL_TAG} from '../member-welcome-emails/constants';
-import * as automationsApi from '../automations/automations-api';
-import {AutomationEmailAnalyticsBatchProcessor} from './automation-email-analytics-batch-processor';
-
-export const newsletters = new EmailAnalyticsServiceWrapper({
-    logName: 'newsletters'
-});
-
-export const automations = new EmailAnalyticsServiceWrapper({
-    logName: 'automations',
-});
-
-export const init = () => {
-    const newsletterEmailEventProcessor = new EmailEventProcessor({
-        domainEvents,
-        db,
-        eventStorage: new NewsletterEmailEventStorage({
-            db,
-            membersRepository: membersService.api.members,
-            models: {
-                Email,
-                EmailRecipientFailure,
-                EmailSpamComplaintEvent
-            },
-            emailSuppressionList,
-            prometheusClient
-        }),
-        prometheusClient
-    });
-
-    const newsletterMailgunTags = ['bulk-email'];
-    if (config.get('bulkEmail:mailgun:tag')) {
-        newsletterMailgunTags.push(config.get('bulkEmail:mailgun:tag'));
-    }
-
-    newsletters.init({
-        event: StartEmailAnalyticsJobEvent,
-        mailgunTags: newsletterMailgunTags,
-        jobNames: {
-            latestNonOpened: 'email-analytics-latest-others',
-            missing: 'email-analytics-missing',
-            latestOpened: 'email-analytics-latest-opened',
-            scheduled: 'email-analytics-scheduled'
-        },
-        cursorSeed: {
-            tableName: 'email_recipients',
-            eventColumns: {
-                delivered: 'delivered_at',
-                opened: 'opened_at',
-                failed: 'failed_at'
-            }
-        },
-        prometheusClient,
-        createEventProcessor: () => (
-            new NewsletterEmailAnalyticsBatchProcessor({
-                config,
-                emailEventProcessor: newsletterEmailEventProcessor,
-                prometheusClient,
-                queries
-            })
-        )
-    });
-
-    automations.init({
-        event: StartAutomationEmailAnalyticsJobEvent,
-        mailgunTags: [AUTOMATION_EMAIL_TAG],
-        jobNames: {
-            latestNonOpened: 'email-analytics-automation-latest-others',
-            missing: 'email-analytics-automation-missing',
-            latestOpened: 'email-analytics-automation-latest-opened',
-            scheduled: 'email-analytics-automation-scheduled'
-        },
-        cursorSeed: {
-            tableName: 'automated_email_recipients',
-            eventColumns: {
-                delivered: 'delivered_at',
-                opened: 'opened_at'
-            }
-        },
-        createEventProcessor: () => (
-            new AutomationEmailAnalyticsBatchProcessor({
-                automationsApi
-            })
-        )
-    });
-};