ghost 5.14.2 → 5.16.0

package/core/server/services/explore/service.js

@@ -46,11 +46,13 @@ module.exports = class ExploreService {
             }
         };
 
-        const mostRecentlyPublishedPost = await this.PostsService.getMostRecentlyPublishedPost();
-        exploreProperties.most_recently_published_at = mostRecentlyPublishedPost?.get('published_at') || null;
+        const mostRecentlyPublishedPost = await this.PostsService.stats.getMostRecentlyPublishedPostDate();
+        const totalPostsPublished = await this.PostsService.stats.getTotalPostsPublished();
+        exploreProperties.most_recently_published_at = mostRecentlyPublishedPost ?? null;
+        exploreProperties.total_posts_published = totalPostsPublished ?? null;
 
         const owner = await this.UserModel.findOne({role: 'Owner', status: 'all'});
-        exploreProperties.owner_email = owner?.get('email') || null;
+        exploreProperties.owner_email = owner?.get('email') ?? null;
 
         return exploreProperties;
     }

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -0,0 +1,88 @@
+const LinkRedirect = require('@tryghost/link-redirects').LinkRedirect;
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkRedirectRepository {
+    /** @type {Object} */
+    #LinkRedirect;
+    /** @type {Object} */
+    #urlUtils;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.LinkRedirect Bookshelf Model
+     * @param {object} deps.urlUtils
+     */
+    constructor(deps) {
+        this.#LinkRedirect = deps.LinkRedirect;
+        this.#urlUtils = deps.urlUtils;
+    }
+
+    /**
+     * @param {InstanceType<LinkRedirect>} linkRedirect 
+     * @returns {Promise<void>}
+     */
+    async save(linkRedirect) {
+        const model = await this.#LinkRedirect.add({
+            // Only store the parthname (no support for variable query strings)
+            from: this.stripSubdirectoryFromPath(linkRedirect.from.pathname),
+            to: linkRedirect.to.href
+        }, {});
+
+        linkRedirect.link_id = ObjectID.createFromHexString(model.id);
+    }
+
+    #trimLeadingSlash(url) {
+        return url.replace(/^\//, '');
+    }
+
+    fromModel(model) {
+        return new LinkRedirect({
+            id: model.id,
+            from: new URL(this.#trimLeadingSlash(model.get('from')), this.#urlUtils.urlFor('home', true)),
+            to: new URL(model.get('to'))
+        });
+    }
+
+    async getAll(options) {
+        const collection = await this.#LinkRedirect.findAll(options);
+
+        const result = [];
+
+        for (const model of collection.models) {
+            result.push(this.fromModel(model));
+        }
+
+        return result;
+    }
+
+    /**
+     * 
+     * @param {URL} url 
+     * @returns {Promise<InstanceType<LinkRedirect>|undefined>} linkRedirect 
+     */
+    async getByURL(url) {
+        // Strip subdirectory from path
+        const from = this.stripSubdirectoryFromPath(url.pathname);
+
+        const linkRedirect = await this.#LinkRedirect.findOne({
+            from
+        }, {});
+
+        if (linkRedirect) {
+            return this.fromModel(linkRedirect);
+        }
+    }
+
+    /**
+     * Convert root relative URLs to subdirectory relative URLs
+    */
+    stripSubdirectoryFromPath(path) {
+        // Bit weird, but only way to do it with the urlUtils atm
+
+        // First convert path to an absolute path
+        const absolute = this.#urlUtils.relativeToAbsolute(path);
+
+        // Then convert it to a relative path, but without subdirectory
+        return this.#urlUtils.absoluteToRelative(absolute, {withoutSubdirectory: true});
+    }
+};

package/core/server/services/link-redirection/index.js

@@ -0,0 +1,31 @@
+const urlUtils = require('../../../shared/url-utils');
+const LinkRedirectRepository = require('./LinkRedirectRepository');
+
+class LinkRedirectsServiceWrapper {
+    async init() {
+        if (this.service) {
+            // Already done
+            return;
+        }
+
+        // Wire up all the dependencies
+        const models = require('../../models');
+
+        const {LinkRedirectsService} = require('@tryghost/link-redirects');
+
+        this.linkRedirectRepository = new LinkRedirectRepository({
+            LinkRedirect: models.LinkRedirect,
+            urlUtils
+        });
+
+        // Expose the service
+        this.service = new LinkRedirectsService({
+            linkRedirectRepository: this.linkRedirectRepository,
+            config: {
+                baseURL: new URL(urlUtils.getSiteUrl())
+            }
+        });
+    }
+}
+
+module.exports = new LinkRedirectsServiceWrapper();

package/core/server/services/link-tracking/LinkClickRepository.js

@@ -0,0 +1,69 @@
+const {LinkClick} = require('@tryghost/link-tracking');
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkClickRepository {
+    /** @type {Object} */
+    #MemberLinkClickEventModel;
+
+    /** @type {Object} */
+    #MemberLinkClickEvent;
+
+    /** @type {object} */
+    #Member;
+
+    /** @type {object} */
+    #DomainEvents;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MemberLinkClickEventModel Bookshelf Model
+     * @param {object} deps.Member Bookshelf Model
+     * @param {object} deps.MemberLinkClickEvent Event
+     * @param {object} deps.DomainEvents
+     */
+    constructor(deps) {
+        this.#MemberLinkClickEventModel = deps.MemberLinkClickEventModel;
+        this.#Member = deps.Member;
+        this.#MemberLinkClickEvent = deps.MemberLinkClickEvent;
+        this.#DomainEvents = deps.DomainEvents;
+    }
+
+    async getAll(options) {
+        const collection = await this.#MemberLinkClickEventModel.findAll(options);
+
+        const result = [];
+
+        for (const model of collection.models) {
+            const member = await this.#Member.findOne({id: model.get('member_id')});
+            result.push(new LinkClick({
+                link_id: model.get('link_id'),
+                member_uuid: member.get('uuid')
+            }));
+        }
+
+        return result;
+    }
+
+    /**
+     * @param {LinkClick} linkClick
+     * @returns {Promise<void>}
+     */
+    async save(linkClick) {
+        // Convert uuid to id
+        const member = await this.#Member.findOne({uuid: linkClick.member_uuid});
+        if (!member) {
+            return;
+        }
+
+        const model = await this.#MemberLinkClickEventModel.add({
+            // Only store the parthname (no support for variable query strings)
+            link_id: linkClick.link_id.toHexString(), 
+            member_id: member.id
+        }, {});
+
+        linkClick.event_id = ObjectID.createFromHexString(model.id);
+
+        // Dispatch event
+        this.#DomainEvents.dispatch(this.#MemberLinkClickEvent.create({memberId: member.id, memberLastSeenAt: member.get('last_seen_at'), linkId: linkClick.link_id.toHexString()}, new Date()));
+    }
+};

package/core/server/services/link-tracking/PostLinkRepository.js

@@ -0,0 +1,62 @@
+const {FullPostLink} = require('@tryghost/link-tracking');
+
+/**
+ * @typedef {import('bson-objectid').default} ObjectID
+ * @typedef {import('@tryghost/link-tracking/lib/PostLink')} PostLink
+ */
+
+module.exports = class PostLinkRepository {
+    /** @type {Object} */
+    #LinkRedirect;
+    /** @type {Object} */
+    #linkRedirectRepository;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.LinkRedirect Bookshelf Model
+     * @param {object} deps.linkRedirectRepository Bookshelf Model
+     */
+    constructor(deps) {
+        this.#LinkRedirect = deps.LinkRedirect;
+        this.#linkRedirectRepository = deps.linkRedirectRepository;
+    }
+
+    /**
+     * 
+     * @param {*} options 
+     * @returns {Promise<InstanceType<FullPostLink>[]>}
+     */
+    async getAll(options) {
+        const collection = await this.#LinkRedirect.findAll({...options, withRelated: ['count.clicks']});
+
+        const result = [];
+
+        for (const model of collection.models) {
+            const link = this.#linkRedirectRepository.fromModel(model);
+
+            result.push(
+                new FullPostLink({
+                    post_id: model.get('post_id'),
+                    link,
+                    count: {
+                        clicks: model.get('count__clicks')
+                    }
+                })
+            );
+        }
+
+        return result;
+    }
+
+    /**
+     * @param {PostLink} postLink
+     * @returns {Promise<void>}
+     */
+    async save(postLink) {
+        await this.#LinkRedirect.edit({
+            post_id: postLink.post_id.toHexString()
+        }, {
+            id: postLink.link_id.toHexString()
+        });
+    }
+};

package/core/server/services/link-tracking/index.js

@@ -0,0 +1,48 @@
+const LinkClickRepository = require('./LinkClickRepository');
+const PostLinkRepository = require('./PostLinkRepository');
+const errors = require('@tryghost/errors');
+
+class LinkTrackingServiceWrapper {
+    async init() {
+        if (this.service) {
+            // Already done
+            return;
+        }
+
+        const linkRedirection = require('../link-redirection');
+        if (!linkRedirection.service) {
+            throw new errors.InternalServerError({message: 'LinkRedirectionService should be initialised before LinkTrackingService'});
+        }
+
+        // Wire up all the dependencies
+        const models = require('../../models');
+        const {MemberLinkClickEvent} = require('@tryghost/member-events');
+        const DomainEvents = require('@tryghost/domain-events');
+
+        const {LinkClickTrackingService} = require('@tryghost/link-tracking');
+
+        const postLinkRepository = new PostLinkRepository({
+            LinkRedirect: models.LinkRedirect,
+            linkRedirectRepository: linkRedirection.linkRedirectRepository
+        });
+
+        const linkClickRepository = new LinkClickRepository({
+            MemberLinkClickEventModel: models.MemberLinkClickEvent,
+            Member: models.Member,
+            MemberLinkClickEvent: MemberLinkClickEvent,
+            DomainEvents
+        });
+
+        // Expose the service
+        this.service = new LinkClickTrackingService({
+            linkRedirectService: linkRedirection.service,
+            linkClickRepository,
+            postLinkRepository,
+            DomainEvents
+        });
+
+        await this.service.init();
+    }
+}
+
+module.exports = new LinkTrackingServiceWrapper();

package/core/server/services/mega/email-preview.js

@@ -27,6 +27,7 @@ class EmailPreview {
             emailContent = postEmailSerializer.renderEmailForSegment(emailContent, memberSegment);
         }
 
+        // Do fake replacements, just like a normal email, but use fallbacks and empty values
         const replacements = postEmailSerializer.parseReplacements(emailContent);
 
         replacements.forEach((replacement) => {
@@ -36,6 +37,12 @@ class EmailPreview {
             );
         });
 
+        // Replace unsubscribe URL (%recipient.unsubscribe_url% replacement)
+        // We should do this only here because replacements should happen at the very end only, just like when an actual email would be send
+        const previewUnsubscribeUrl = postEmailSerializer.createUnsubscribeUrl(null);
+        emailContent.html = emailContent.html.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+        emailContent.plaintext = emailContent.plaintext.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+
         return {
             subject: emailContent.subject,
             html: emailContent.html,

package/core/server/services/mega/mega.js

@@ -85,7 +85,7 @@ const getEmailData = async (postModel, options) => {
  * @param {ValidMemberSegment} [memberSegment]
  */
 const sendTestEmail = async (postModel, toEmails, memberSegment) => {
-    let emailData = await getEmailData(postModel);
+    let emailData = await getEmailData(postModel, {isTestEmail: true});
     emailData.subject = `[Test] ${emailData.subject}`;
 
     // fetch any matching members so that replacements use expected values

package/core/server/services/mega/post-email-serializer.js

@@ -14,8 +14,11 @@ const {isUnsplashImage, isLocalContentImage} = require('@tryghost/kg-default-car
 const {textColorForBackgroundColor, darkenToContrastThreshold} = require('@tryghost/color-utils');
 const logging = require('@tryghost/logging');
 const urlService = require('../../services/url');
+const linkReplacer = require('@tryghost/link-replacer');
+const linkTracking = require('../link-tracking');
+const memberAttribution = require('../member-attribution');
 
-const ALLOWED_REPLACEMENTS = ['first_name'];
+const ALLOWED_REPLACEMENTS = ['first_name', 'uuid'];
 
 const PostEmailSerializer = {
     
@@ -243,7 +246,7 @@ const PostEmailSerializer = {
         return templateSettings;
     },
 
-    async serialize(postModel, newsletter, options = {isBrowserPreview: false}) {
+    async serialize(postModel, newsletter, options = {isBrowserPreview: false, isTestEmail: false}) {
         const post = await this.serializePostModel(postModel);
 
         const timezone = settingsCache.get('timezone');
@@ -276,7 +279,7 @@ const PostEmailSerializer = {
         // perform any email specific adjustments to the mobiledoc->HTML render output
         // body wrapper is required so we can get proper top-level selections
         const cheerio = require('cheerio');
-        let _cheerio = cheerio.load(`<body>${post.html}</body>`);
+        const _cheerio = cheerio.load(`<body>${post.html}</body>`);
         // remove leading/trailing HRs
         _cheerio(`
             body > hr:first-child,
@@ -284,8 +287,10 @@ const PostEmailSerializer = {
             body > div:first-child > hr:first-child,
             body > div:last-child > hr:last-child
         `).remove();
-        post.html = _cheerio('body').html();
+        post.html = _cheerio('body').html(); // () (added this comment because of a bug in the syntax highlighter in VSCode)
 
+        // Note: we don't need to do link replacements on the plaintext here
+        // because the plaintext will get recalculated on the updated post html (which already includes link replacements) in renderEmailForSegment
         post.plaintext = htmlToPlaintext.email(post.html);
 
         // Outlook will render feature images at full-size breaking the layout.
@@ -325,24 +330,73 @@ const PostEmailSerializer = {
 
         let htmlTemplate = render({post, site: this.getSite(), templateSettings, newsletter: newsletter.toJSON()});
 
-        if (options.isBrowserPreview) {
-            const previewUnsubscribeUrl = this.createUnsubscribeUrl(null);
-            htmlTemplate = htmlTemplate.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+        // The plaintext version that is returned here is actually never really used for sending because we'll use htmlToPlaintext again later
+        let result = {
+            html: this.formatHtmlForEmail(htmlTemplate),
+            plaintext: post.plaintext
+        };
+
+        /** 
+         *  If a part of the email is members-only and the post is paid-only, add a paywall:
+         *  - Just before sending the email, we'll hide the paywall or paid content depending on the member segment it is sent to.
+         *  - We already need to do URL-replacement on the HTML here
+         *  - Link replacement cannot happen later because renderEmailForSegment is called multiple times for a single email (which would result in duplicate redirects)
+        */
+        const isPaidPost = post.visibility === 'paid' || post.visibility === 'tiers';
+
+        const paywallIndex = (result.html || '').indexOf('<!--members-only-->');
+        if (paywallIndex !== -1 && isPaidPost) {
+            const postContentEndIdx = result.html.indexOf('<!-- POST CONTENT END -->');
+
+            if (postContentEndIdx !== -1) {
+                const paywallHTML = '<!-- PAYWALL -->' + this.renderPaywallCTA(post);
+
+                // Append it just before the end of the post content
+                result.html = result.html.slice(0, postContentEndIdx) + paywallHTML + result.html.slice(postContentEndIdx);
+            }
+        }
+
+        // Now replace the links in the HTML version
+        if (labs.isSet('emailClicks')) {
+            if ((!options.isBrowserPreview && !options.isTestEmail) || process.env.NODE_ENV === 'development') {
+                const enableTracking = settingsCache.get('email_track_clicks');
+                result.html = await linkReplacer.replace(result.html, async (url) => {
+                    // Add newsletter source attribution
+                    url = memberAttribution.service.addEmailSourceAttributionTracking(url, newsletter);
+                    const isSite = urlUtils.isSiteUrl(url);
+
+                    // Add post attribution tracking
+                    if (isSite && enableTracking) {
+                        // Only add attribution links to our own site (except for the newsletter referrer)
+                        url = memberAttribution.service.addPostAttributionTracking(url, post);
+                    }
+
+                    // Add link click tracking
+                    if (enableTracking) {
+                        url = await linkTracking.service.addTrackingToUrl(url, post, '--uuid--');
+                        
+                        // We need to convert to a string at this point, because we need invalid string characters in the URL
+                        const str = url.toString().replace(/--uuid--/g, '%%{uuid}%%');
+                        return str;
+                    }
+
+                    // Replace the URL with a normal redirect so we can change it later, but don't include tracking
+                    url = linkTracking.service.addRedirectToUrl(url, post);
+                    return url;
+                });
+            }
         }
 
         // Clean up any unknown replacements strings to get our final content
-        const {html, plaintext} = this.normalizeReplacementStrings({
-            html: this.formatHtmlForEmail(htmlTemplate),
-            plaintext: post.plaintext
-        });
+        const {html, plaintext} = this.normalizeReplacementStrings(result);
         const data = {
             subject: post.email_subject || post.title,
             html,
             plaintext
         };
-        if (labs.isSet('newsletterPaywall')) {
-            data.post = post;
-        }
+
+        // Add post for checking access in renderEmailForSegment (only for previews)
+        data.post = post;
         return data;
     },
 
@@ -392,25 +446,45 @@ const PostEmailSerializer = {
 
         const result = {...email};
 
-        /** Checks and hides content for newsletter behind paywall card
-         *  based on member's status and post access
-         *  Adds CTA in case content is hidden.
-        */
-        if (labs.isSet('newsletterPaywall')) {
-            const paywallIndex = (result.html || '').indexOf('<!--members-only-->');
-            if (paywallIndex !== -1 && memberSegment && result.post) {
+        // Note about link tracking:
+        // Don't add new HTML in here, but add it in the serialize method and surround it with the required HTML comments or attributes
+        // This is because we can't replace links at this point (this is executed multiple times, once per batch and we don't want to generate duplicate links for the same email)
+
+        // Remove the paywall or members-only content based on the current member segment
+        const startMembersOnlyContent = (result.html || '').indexOf('<!--members-only-->');
+        const startPaywall = result.html.indexOf('<!-- PAYWALL -->');
+        let endPost = result.html.indexOf('<!-- POST CONTENT END -->');
+
+        if (endPost === -1) {
+            // Default to the end of the HTML (shouldn't happen, but just in case if we have members-only content that should get removed)
+            endPost = result.html.length;
+        }
+
+        // We support the cases where there is no <!--members-only--> but there is a paywall (in case of bugs)
+        // We also support the case where there is no <!-- PAYWALL --> but there is a <!--members-only--> (in case of bugs)
+        if (startMembersOnlyContent !== -1 || startPaywall !== -1) {
+            // By default remove the paywall if no memberSegment is passed
+            let memberHasAccess = true;
+
+            if (memberSegment && result.post) {
                 let statusFilter = memberSegment === 'status:free' ? {status: 'free'} : {status: 'paid'};
                 const postVisiblity = result.post.visibility;
 
                 // For newsletter paywall, specific tiers visibility is considered on par to paid tiers
                 result.post.visibility = postVisiblity === 'tiers' ? 'paid' : postVisiblity;
 
-                const memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+                memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+            }
 
-                if (!memberHasAccess) {
-                    const postContentEndIdx = result.html.search(/[\s\n\r]+?<!-- POST CONTENT END -->/);
-                    result.html = result.html.slice(0, paywallIndex) + this.renderPaywallCTA(result.post) + result.html.slice(postContentEndIdx);
-                    result.plaintext = htmlToPlaintext.excerpt(result.html);
+            if (!memberHasAccess) {
+                if (startMembersOnlyContent !== -1) {
+                    // Remove the members-only content, but keep the paywall (if there is a paywall)
+                    result.html = result.html.slice(0, startMembersOnlyContent) + result.html.slice(startPaywall === -1 ? endPost : startPaywall);
+                }
+            } else {
+                if (startPaywall !== -1) {
+                    // Remove the paywall
+                    result.html = result.html.slice(0, startPaywall) + result.html.slice(endPost);
                 }
             }
         }
@@ -427,7 +501,7 @@ const PostEmailSerializer = {
         });
 
         result.html = this.formatHtmlForEmail($.html());
-        result.plaintext = htmlToPlaintext.email(result.html);
+        result.plaintext = htmlToPlaintext.email(result.html); 
         delete result.post;
 
         return result;

package/core/server/services/member-attribution/index.js

@@ -9,20 +9,27 @@ class MemberAttributionServiceWrapper {
         }
 
         // Wire up all the dependencies
-        const {MemberAttributionService, UrlTranslator, AttributionBuilder} = require('@tryghost/member-attribution');
+        const {
+            MemberAttributionService, UrlTranslator, ReferrerTranslator, AttributionBuilder
+        } = require('@tryghost/member-attribution');
         const models = require('../../models');
 
         const urlTranslator = new UrlTranslator({
-            urlService, 
+            urlService,
             urlUtils,
             models: {
-                Post: models.Post, 
-                User: models.User, 
+                Post: models.Post,
+                User: models.User,
                 Tag: models.Tag
             }
         });
 
-        this.attributionBuilder = new AttributionBuilder({urlTranslator});
+        const referrerTranslator = new ReferrerTranslator({
+            siteUrl: urlUtils.urlFor('home', true),
+            adminUrl: urlUtils.urlFor('admin', true)
+        });
+
+        this.attributionBuilder = new AttributionBuilder({urlTranslator, referrerTranslator});
 
         // Expose the service
         this.service = new MemberAttributionService({

package/core/server/services/members/api.js

@@ -13,7 +13,6 @@ const SingleUseTokenProvider = require('./SingleUseTokenProvider');
 const urlUtils = require('../../../shared/url-utils');
 const labsService = require('../../../shared/labs');
 const offersService = require('../offers');
-const staffService = require('../staff');
 const newslettersService = require('../newsletters');
 const memberAttributionService = require('../member-attribution');
 
@@ -188,6 +187,7 @@ function createApiInstance(config) {
             MemberAnalyticEvent: models.MemberAnalyticEvent,
             MemberCreatedEvent: models.MemberCreatedEvent,
             SubscriptionCreatedEvent: models.SubscriptionCreatedEvent,
+            MemberLinkClickEvent: models.MemberLinkClickEvent,
             OfferRedemption: models.OfferRedemption,
             Offer: models.Offer,
             StripeProduct: models.StripeProduct,
@@ -198,7 +198,6 @@ function createApiInstance(config) {
         },
         stripeAPIService: stripeService.api,
         offersAPI: offersService.api,
-        staffService: staffService.api,
         labsService: labsService,
         newslettersService: newslettersService,
         memberAttributionService: memberAttributionService.service

package/core/server/services/permissions/index.js

@@ -19,6 +19,5 @@ module.exports = {
     init: init,
     canThis: require('./can-this'),
     // @TODO: Make it so that we don't need to export these
-    parseContext: require('./parse-context'),
-    applyPublicRules: require('./public')
+    parseContext: require('./parse-context')
 };