ghost 5.14.1 → 5.15.0

package/core/server/services/explore/service.js

@@ -46,11 +46,13 @@ module.exports = class ExploreService {
             }
         };
 
-        const mostRecentlyPublishedPost = await this.PostsService.getMostRecentlyPublishedPost();
-        exploreProperties.most_recently_published_at = mostRecentlyPublishedPost?.get('published_at') || null;
+        const mostRecentlyPublishedPost = await this.PostsService.stats.getMostRecentlyPublishedPostDate();
+        const totalPostsPublished = await this.PostsService.stats.getTotalPostsPublished();
+        exploreProperties.most_recently_published_at = mostRecentlyPublishedPost ?? null;
+        exploreProperties.total_posts_published = totalPostsPublished ?? null;
 
         const owner = await this.UserModel.findOne({role: 'Owner', status: 'all'});
-        exploreProperties.owner_email = owner?.get('email') || null;
+        exploreProperties.owner_email = owner?.get('email') ?? null;
 
         return exploreProperties;
     }

package/core/server/services/link-click-tracking/index.js

@@ -0,0 +1,25 @@
+class LinkTrackingServiceWrapper {
+    async init() {
+        if (this.service) {
+            // Already done
+            return;
+        }
+
+        // Wire up all the dependencies
+        const LinkTrackingService = require('@tryghost/link-tracking');
+
+        // Expose the service
+        this.service = new LinkTrackingService({
+            linkClickRepository: {
+                async save(linkClick) {
+                    // eslint-disable-next-line no-console
+                    console.log('Saving link click', linkClick);
+                }
+            }
+        });
+
+        await this.service.init();
+    }
+}
+
+module.exports = new LinkTrackingServiceWrapper();

package/core/server/services/link-redirection/index.js

@@ -0,0 +1,33 @@
+const urlUtils = require('../../../shared/url-utils');
+
+class LinkRedirectsServiceWrapper {
+    async init() {
+        if (this.service) {
+            // Already done
+            return;
+        }
+
+        // Wire up all the dependencies
+        const {LinkRedirectsService} = require('@tryghost/link-redirects');
+
+        const store = [];
+        // Expose the service
+        this.service = new LinkRedirectsService({
+            linkRedirectRepository: {
+                async save(linkRedirect) {
+                    store.push(linkRedirect);
+                },
+                async getByURL(url) {
+                    return store.find((link) => {
+                        return link.from.pathname === url.pathname;
+                    });
+                }
+            },
+            config: {
+                baseURL: new URL(urlUtils.getSiteUrl())
+            }
+        });
+    }
+}
+
+module.exports = new LinkRedirectsServiceWrapper();

package/core/server/services/link-replacement/index.js

@@ -0,0 +1,24 @@
+class LinkReplacementServiceWrapper {
+    init() {
+        if (this.service) {
+            // Already done
+            return;
+        }
+
+        // Wire up all the dependencies
+        const LinkReplacementService = require('@tryghost/link-replacement');
+        const urlUtils = require('../../../shared/url-utils');
+        const settingsCache = require('../../../shared/settings-cache');
+
+        // Expose the service
+        this.service = new LinkReplacementService({
+            linkRedirectService: require('../link-redirection').service,
+            linkClickTrackingService: require('../link-click-tracking').service,
+            attributionService: require('../member-attribution').service,
+            urlUtils,
+            settingsCache
+        });
+    }
+}
+
+module.exports = new LinkReplacementServiceWrapper();

package/core/server/services/mega/email-preview.js

@@ -27,6 +27,7 @@ class EmailPreview {
             emailContent = postEmailSerializer.renderEmailForSegment(emailContent, memberSegment);
         }
 
+        // Do fake replacements, just like a normal email, but use fallbacks and empty values
         const replacements = postEmailSerializer.parseReplacements(emailContent);
 
         replacements.forEach((replacement) => {
@@ -36,6 +37,12 @@ class EmailPreview {
             );
         });
 
+        // Replace unsubscribe URL (%recipient.unsubscribe_url% replacement)
+        // We should do this only here because replacements should happen at the very end only, just like when an actual email would be send
+        const previewUnsubscribeUrl = postEmailSerializer.createUnsubscribeUrl(null);
+        emailContent.html = emailContent.html.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+        emailContent.plaintext = emailContent.plaintext.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+
         return {
             subject: emailContent.subject,
             html: emailContent.html,

package/core/server/services/mega/mega.js

@@ -85,7 +85,7 @@ const getEmailData = async (postModel, options) => {
  * @param {ValidMemberSegment} [memberSegment]
  */
 const sendTestEmail = async (postModel, toEmails, memberSegment) => {
-    let emailData = await getEmailData(postModel);
+    let emailData = await getEmailData(postModel, {isTestEmail: true});
     emailData.subject = `[Test] ${emailData.subject}`;
 
     // fetch any matching members so that replacements use expected values

package/core/server/services/mega/post-email-serializer.js

@@ -14,8 +14,9 @@ const {isUnsplashImage, isLocalContentImage} = require('@tryghost/kg-default-car
 const {textColorForBackgroundColor, darkenToContrastThreshold} = require('@tryghost/color-utils');
 const logging = require('@tryghost/logging');
 const urlService = require('../../services/url');
+const linkReplacement = require('../link-replacement');
 
-const ALLOWED_REPLACEMENTS = ['first_name'];
+const ALLOWED_REPLACEMENTS = ['first_name', 'uuid'];
 
 const PostEmailSerializer = {
     
@@ -243,7 +244,7 @@ const PostEmailSerializer = {
         return templateSettings;
     },
 
-    async serialize(postModel, newsletter, options = {isBrowserPreview: false}) {
+    async serialize(postModel, newsletter, options = {isBrowserPreview: false, isTestEmail: false}) {
         const post = await this.serializePostModel(postModel);
 
         const timezone = settingsCache.get('timezone');
@@ -276,7 +277,7 @@ const PostEmailSerializer = {
         // perform any email specific adjustments to the mobiledoc->HTML render output
         // body wrapper is required so we can get proper top-level selections
         const cheerio = require('cheerio');
-        let _cheerio = cheerio.load(`<body>${post.html}</body>`);
+        const _cheerio = cheerio.load(`<body>${post.html}</body>`);
         // remove leading/trailing HRs
         _cheerio(`
             body > hr:first-child,
@@ -284,8 +285,10 @@ const PostEmailSerializer = {
             body > div:first-child > hr:first-child,
             body > div:last-child > hr:last-child
         `).remove();
-        post.html = _cheerio('body').html();
+        post.html = _cheerio('body').html(); // () (added this comment because of a bug in the syntax highlighter in VSCode)
 
+        // Note: we don't need to do link replacements on the plaintext here
+        // because the plaintext will get recalculated on the updated post html (which already includes link replacements) in renderEmailForSegment
         post.plaintext = htmlToPlaintext.email(post.html);
 
         // Outlook will render feature images at full-size breaking the layout.
@@ -325,24 +328,49 @@ const PostEmailSerializer = {
 
         let htmlTemplate = render({post, site: this.getSite(), templateSettings, newsletter: newsletter.toJSON()});
 
-        if (options.isBrowserPreview) {
-            const previewUnsubscribeUrl = this.createUnsubscribeUrl(null);
-            htmlTemplate = htmlTemplate.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
+        // The plaintext version that is returned here is actually never really used for sending because we'll use htmlToPlaintext again later
+        let result = {
+            html: this.formatHtmlForEmail(htmlTemplate),
+            plaintext: post.plaintext
+        };
+
+        /** 
+         *  If a part of the email is members-only and the post is paid-only, add a paywall:
+         *  - Just before sending the email, we'll hide the paywall or paid content depending on the member segment it is sent to.
+         *  - We already need to do URL-replacement on the HTML here
+         *  - Link replacement cannot happen later because renderEmailForSegment is called multiple times for a single email (which would result in duplicate redirects)
+        */
+        const isPaidPost = post.visibility === 'paid' || post.visibility === 'tiers';
+
+        const paywallIndex = (result.html || '').indexOf('<!--members-only-->');
+        if (paywallIndex !== -1 && isPaidPost) {
+            const postContentEndIdx = result.html.indexOf('<!-- POST CONTENT END -->');
+
+            if (postContentEndIdx !== -1) {
+                const paywallHTML = '<!-- PAYWALL -->' + this.renderPaywallCTA(post);
+
+                // Append it just before the end of the post content
+                result.html = result.html.slice(0, postContentEndIdx) + paywallHTML + result.html.slice(postContentEndIdx);
+            }
+        }
+
+        // Now replace the links in the HTML version
+        if (labs.isSet('emailClicks')) {
+            if ((!options.isBrowserPreview && !options.isTestEmail) || process.env.NODE_ENV === 'development') {
+                result.html = await linkReplacement.service.replaceLinks(result.html, newsletter, postModel);
+            }
         }
 
         // Clean up any unknown replacements strings to get our final content
-        const {html, plaintext} = this.normalizeReplacementStrings({
-            html: this.formatHtmlForEmail(htmlTemplate),
-            plaintext: post.plaintext
-        });
+        const {html, plaintext} = this.normalizeReplacementStrings(result);
         const data = {
             subject: post.email_subject || post.title,
             html,
             plaintext
         };
-        if (labs.isSet('newsletterPaywall')) {
-            data.post = post;
-        }
+
+        // Add post for checking access in renderEmailForSegment (only for previews)
+        data.post = post;
         return data;
     },
 
@@ -392,25 +420,45 @@ const PostEmailSerializer = {
 
         const result = {...email};
 
-        /** Checks and hides content for newsletter behind paywall card
-         *  based on member's status and post access
-         *  Adds CTA in case content is hidden.
-        */
-        if (labs.isSet('newsletterPaywall')) {
-            const paywallIndex = (result.html || '').indexOf('<!--members-only-->');
-            if (paywallIndex !== -1 && memberSegment && result.post) {
+        // Note about link tracking:
+        // Don't add new HTML in here, but add it in the serialize method and surround it with the required HTML comments or attributes
+        // This is because we can't replace links at this point (this is executed multiple times, once per batch and we don't want to generate duplicate links for the same email)
+
+        // Remove the paywall or members-only content based on the current member segment
+        const startMembersOnlyContent = (result.html || '').indexOf('<!--members-only-->');
+        const startPaywall = result.html.indexOf('<!-- PAYWALL -->');
+        let endPost = result.html.indexOf('<!-- POST CONTENT END -->');
+
+        if (endPost === -1) {
+            // Default to the end of the HTML (shouldn't happen, but just in case if we have members-only content that should get removed)
+            endPost = result.html.length;
+        }
+
+        // We support the cases where there is no <!--members-only--> but there is a paywall (in case of bugs)
+        // We also support the case where there is no <!-- PAYWALL --> but there is a <!--members-only--> (in case of bugs)
+        if (startMembersOnlyContent !== -1 || startPaywall !== -1) {
+            // By default remove the paywall if no memberSegment is passed
+            let memberHasAccess = true;
+
+            if (memberSegment && result.post) {
                 let statusFilter = memberSegment === 'status:free' ? {status: 'free'} : {status: 'paid'};
                 const postVisiblity = result.post.visibility;
 
                 // For newsletter paywall, specific tiers visibility is considered on par to paid tiers
                 result.post.visibility = postVisiblity === 'tiers' ? 'paid' : postVisiblity;
 
-                const memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+                memberHasAccess = membersService.contentGating.checkPostAccess(result.post, statusFilter);
+            }
 
-                if (!memberHasAccess) {
-                    const postContentEndIdx = result.html.search(/[\s\n\r]+?<!-- POST CONTENT END -->/);
-                    result.html = result.html.slice(0, paywallIndex) + this.renderPaywallCTA(result.post) + result.html.slice(postContentEndIdx);
-                    result.plaintext = htmlToPlaintext.excerpt(result.html);
+            if (!memberHasAccess) {
+                if (startMembersOnlyContent !== -1) {
+                    // Remove the members-only content, but keep the paywall (if there is a paywall)
+                    result.html = result.html.slice(0, startMembersOnlyContent) + result.html.slice(startPaywall === -1 ? endPost : startPaywall);
+                }
+            } else {
+                if (startPaywall !== -1) {
+                    // Remove the paywall
+                    result.html = result.html.slice(0, startPaywall) + result.html.slice(endPost);
                 }
             }
         }
@@ -427,7 +475,7 @@ const PostEmailSerializer = {
         });
 
         result.html = this.formatHtmlForEmail($.html());
-        result.plaintext = htmlToPlaintext.email(result.html);
+        result.plaintext = htmlToPlaintext.email(result.html); 
         delete result.post;
 
         return result;

package/core/server/services/mega/template.js

@@ -26,7 +26,7 @@ const sanitizeKeys = (obj, keys) => {
 module.exports = ({post, site, newsletter, templateSettings}) => {
     const date = new Date();
     const hasFeatureImageCaption = templateSettings.showFeatureImage && post.feature_image && post.feature_image_caption;
-    const cleanPost = sanitizeKeys(post, ['url', 'published_at', 'title', 'excerpt', 'authors', 'feature_image', 'feature_image_alt', 'feature_image_caption']);
+    const cleanPost = sanitizeKeys(post, ['url', 'published_at', 'title', 'excerpt', 'authors', 'feature_image', 'feature_image_alt']);
     const cleanSite = sanitizeKeys(site, ['title']);
     const cleanNewsletter = sanitizeKeys(newsletter, ['name']);
 

package/core/server/services/members/api.js

@@ -13,7 +13,6 @@ const SingleUseTokenProvider = require('./SingleUseTokenProvider');
 const urlUtils = require('../../../shared/url-utils');
 const labsService = require('../../../shared/labs');
 const offersService = require('../offers');
-const staffService = require('../staff');
 const newslettersService = require('../newsletters');
 const memberAttributionService = require('../member-attribution');
 
@@ -198,7 +197,6 @@ function createApiInstance(config) {
         },
         stripeAPIService: stripeService.api,
         offersAPI: offersService.api,
-        staffService: staffService.api,
         labsService: labsService,
         newslettersService: newslettersService,
         memberAttributionService: memberAttributionService.service

package/core/server/services/permissions/index.js

@@ -19,6 +19,5 @@ module.exports = {
     init: init,
     canThis: require('./can-this'),
     // @TODO: Make it so that we don't need to export these
-    parseContext: require('./parse-context'),
-    applyPublicRules: require('./public')
+    parseContext: require('./parse-context')
 };

package/core/server/services/posts/posts-service.js

@@ -8,11 +8,12 @@ const messages = {
 };
 
 class PostsService {
-    constructor({mega, urlUtils, models, isSet}) {
+    constructor({mega, urlUtils, models, isSet, stats}) {
         this.mega = mega;
         this.urlUtils = urlUtils;
         this.models = models;
         this.isSet = isSet;
+        this.stats = stats;
     }
 
     async editPost(frame) {
@@ -76,20 +77,6 @@ class PostsService {
         }
     }
 
-    /**
-     * Returns the most recently `published_at` post that was published or sent
-     * via email
-     */
-    async getMostRecentlyPublishedPost() {
-        const recentlyPublishedPost = await this.models.Post.findPage({
-            status: ['published', 'sent'],
-            order: 'published_at DESC',
-            limit: 1
-        });
-
-        return recentlyPublishedPost?.data[0];
-    }
-
     /**
      * Calculates if the email should be tried to be sent out
      * @private
@@ -135,12 +122,16 @@ const getPostServiceInstance = () => {
     const {mega} = require('../mega');
     const labs = require('../../../shared/labs');
     const models = require('../../models');
+    const PostStats = require('./stats/post-stats');
+
+    const postStats = new PostStats();
 
     return new PostsService({
         mega: mega,
         urlUtils: urlUtils,
         models: models,
-        isSet: labs.isSet.bind(labs)
+        isSet: labs.isSet.bind(labs),
+        stats: postStats
     });
 };
 

package/core/server/services/posts/stats/post-stats.js

@@ -0,0 +1,35 @@
+const db = require('../../../data/db');
+class PostStats {
+    #db;
+
+    constructor() {
+        this.#db = db;
+    }
+
+    /**
+     * Returns the most recently `published_at` post that was published or sent
+     * via email
+     */
+    async getMostRecentlyPublishedPostDate() {
+        const result = await this.#db.knex.select('published_at')
+            .from('posts')
+            .whereIn('status', ['sent', 'published'])
+            .orderBy('published_at', 'desc')
+            .limit(1);
+
+        return result?.[0]?.published_at ? new Date(result?.[0]?.published_at) : null;
+    }
+
+    /**
+     * Fetches count of all published posts
+     */
+    async getTotalPostsPublished() {
+        const [result] = await this.#db.knex('posts')
+            .whereIn('status', ['sent', 'published'])
+            .count('id', {as: 'total'});
+
+        return result.total;
+    }
+}
+
+module.exports = PostStats;

package/core/server/services/staff/index.js

@@ -1,5 +1,11 @@
+const DomainEvents = require('@tryghost/domain-events');
 class StaffServiceWrapper {
     init() {
+        if (this.api) {
+            // Prevent creating duplicate DomainEvents subscribers
+            return;
+        }
+
         const StaffService = require('@tryghost/staff-service');
 
         const logging = require('@tryghost/logging');
@@ -16,8 +22,11 @@ class StaffServiceWrapper {
             mailer,
             settingsHelpers,
             settingsCache,
-            urlUtils
+            urlUtils,
+            DomainEvents
         });
+
+        this.api.subscribeEvents();
     }
 }
 

package/core/server/services/url/config.js

@@ -12,6 +12,7 @@ module.exports = [
             exclude: [
                 'title',
                 'mobiledoc',
+                'lexical',
                 'html',
                 'plaintext',
                 // @TODO: https://github.com/TryGhost/Ghost/issues/10335
@@ -55,6 +56,7 @@ module.exports = [
             exclude: [
                 'title',
                 'mobiledoc',
+                'lexical',
                 'html',
                 'plaintext',
                 // @TODO: https://github.com/TryGhost/Ghost/issues/10335

package/core/shared/config/defaults.json

@@ -153,8 +153,8 @@
         "version": "0.10.1"
     },
     "editor": {
-        "url": "https://unpkg.com/@tryghost/koenig-react/dist/umd/koenig-react.min.js",
-        "lexicalUrl": "https://unpkg.com/@tryghost/koenig-lexical-experiment/dist/koenig-lexical.umd.js"
+        "url": "https://unpkg.com/@tryghost/koenig-lexical@~{version}/dist/koenig-lexical.umd.js",
+        "version": "0.0"
     },
     "tenor": {
         "googleApiKey": null,

package/core/shared/config/overrides.json

@@ -30,7 +30,7 @@
             "contentTypes": ["image/jpeg", "image/png", "image/gif", "image/svg+xml", "image/x-icon", "image/vnd.microsoft.icon", "image/webp"]
         },
         "media": {
-            "extensions": [".mp4",".webm", ".ogv", ".mp3", ".wav", ".ogg"],
+            "extensions": [".mp4",".webm", ".ogv", ".mp3", ".wav", ".ogg", ".m4a"],
             "contentTypes": [
                 "video/mp4",
                 "video/webm",
@@ -40,7 +40,8 @@
                 "audio/wave",
                 "audio/wav",
                 "audio/x-wav",
-                "audio/ogg"
+                "audio/ogg",
+                "audio/x-m4a"
             ]
         },
         "thumbnails": {

package/core/shared/labs.js

@@ -15,7 +15,6 @@ const messages = {
 
 // flags in this list always return `true`, allows quick global enable prior to full flag removal
 const GA_FEATURES = [
-    'auditLog',
     'newsletterPaywall',
     'freeTrial',
     'compExpiring',
@@ -32,7 +31,10 @@ const BETA_FEATURES = [
 
 const ALPHA_FEATURES = [
     'urlCache',
-    'beforeAfterCard'
+    'beforeAfterCard',
+    'emailClicks',
+    'sourceAttribution',
+    'lexicalEditor'
 ];
 
 module.exports.GA_KEYS = [...GA_FEATURES];