ghost 5.30.1 → 5.32.0

package/core/server/data/migrations/versions/5.31/2023-01-17-14-59-add-outbound-link-tagging-setting.js

@@ -0,0 +1,8 @@
+const {addSetting} = require('../../utils');
+
+module.exports = addSetting({
+    key: 'outbound_link_tagging',
+    value: 'true',
+    type: 'boolean',
+    group: 'analytics'
+});

package/core/server/data/migrations/versions/5.31/2023-01-19-07-46-add-mentions-table.js

@@ -0,0 +1,17 @@
+const {addTable} = require('../../utils');
+
+module.exports = addTable('mentions', {
+    id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+    source: {type: 'string', maxlength: 2000, nullable: false},
+    source_title: {type: 'string', maxlength: 2000, nullable: true},
+    source_site_title: {type: 'string', maxlength: 2000, nullable: true},
+    source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
+    source_author: {type: 'string', maxlength: 2000, nullable: true},
+    source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
+    source_favicon: {type: 'string', maxlength: 2000, nullable: true},
+    target: {type: 'string', maxlength: 2000, nullable: false},
+    resource_id: {type: 'string', maxlength: 24, nullable: true},
+    resource_type: {type: 'string', maxlength: 50, nullable: true},
+    created_at: {type: 'dateTime', nullable: false},
+    payload: {type: 'text', maxlength: 65535, nullable: true}
+});

package/core/server/data/migrations/versions/5.32/2023-01-24-08-00-fix-invalid-tier-expiry-for-paid-members.js

@@ -0,0 +1,35 @@
+const logging = require('@tryghost/logging');
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Removing expiry dates for paid members');
+        try {
+            // Fetch all members with a paid status that have an expiry date
+            // Paid members should not have an expiry date
+            const invalidExpiryIds = await knex('members_products')
+                .select('members_products.id')
+                .leftJoin('members', 'members_products.member_id', 'members.id')
+                .where('members.status', '=', 'paid')
+                .whereNotNull('members_products.expiry_at').pluck('members_products.id');
+
+            logging.info(`Found ${invalidExpiryIds.length} paid members with expiry dates`);
+
+            if (invalidExpiryIds.length === 0) {
+                return;
+            }
+
+            logging.info(`Removing expiry dates for ${invalidExpiryIds.length} paid members`);
+
+            await knex('members_products')
+                .update('expiry_at', null)
+                .whereIn('id', invalidExpiryIds);
+        } catch (err) {
+            logging.warn('Failed to remove expiry dates for paid members');
+            logging.warn(err);
+        }
+    },
+    async function down() {
+        // no-op: we don't want to reintroduce the incorrect expiry dates for member tiers
+    }
+);

package/core/server/data/migrations/versions/5.32/2023-01-24-08-09-restore-incorrect-expired-tiers-for-members.js

@@ -0,0 +1,46 @@
+const logging = require('@tryghost/logging');
+const ObjectId = require('bson-objectid').default;
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Restoring member<>tier mapping for members with paid status');
+        try {
+            // fetch all members with a paid status that don't have a members_products record
+            // and have a members_product_events record with an action of "added"
+            // and fetch the product_id from the most recent record for that member
+            const memberWithTiers = await knex.select('m.id as member_id', 'mpe.product_id as product_id')
+                .from('members as m')
+                .leftJoin('members_products as mp', 'm.id', 'mp.member_id')
+                .leftJoin('members_product_events as mpe', function () {
+                    this.on('m.id', 'mpe.member_id')
+                        .andOn(knex.raw('mpe.created_at = (SELECT max(created_at) FROM members_product_events WHERE member_id = mpe.member_id and action = "added")'));
+                })
+                .where({'m.status': 'paid', 'mp.member_id': null, 'mpe.action': 'added'});
+
+            // create a new members_products record for each member with id, member_id and product_id
+            const toInsert = memberWithTiers.map((memberTier) => {
+                return {
+                    ...memberTier,
+                    id: ObjectId().toHexString()
+                };
+            }).filter((memberTier) => {
+                // filter out any members that don't have a product_id for some reason
+                if (!memberTier.product_id) {
+                    logging.warn(`Invalid record found - member_id: ${memberTier.member_id} is without product_id`);
+                    return false;
+                }
+                return true;
+            });
+
+            logging.info(`Inserting ${toInsert.length} records into members_products`);
+            await knex.batchInsert('members_products', toInsert);
+        } catch (err) {
+            logging.warn('Failed to restore member<>tier mapping for members with paid status');
+            logging.warn(err);
+        }
+    },
+    async function down() {
+        // np-op: we don't want to delete the missing records we've just inserted
+    }
+);

package/core/server/data/schema/default-settings/default-settings.json

@@ -480,5 +480,15 @@
                 ]]
             }
         }
+    },
+    "analytics": {
+        "outbound_link_tagging": {
+            "defaultValue": "true",
+            "validations": {
+                "isEmpty": false,
+                "isIn": [["true", "false"]]
+            },
+            "type": "boolean"
+        }
     }
 }

package/core/server/data/schema/schema.js

@@ -979,5 +979,20 @@ module.exports = {
         '@@UNIQUE_CONSTRAINTS@@': [
             ['email_id', 'member_id']
         ]
+    },
+    mentions: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        source: {type: 'string', maxlength: 2000, nullable: false},
+        source_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_site_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
+        source_author: {type: 'string', maxlength: 2000, nullable: true},
+        source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
+        source_favicon: {type: 'string', maxlength: 2000, nullable: true},
+        target: {type: 'string', maxlength: 2000, nullable: false},
+        resource_id: {type: 'string', maxlength: 24, nullable: true},
+        resource_type: {type: 'string', maxlength: 50, nullable: true},
+        created_at: {type: 'dateTime', nullable: false},
+        payload: {type: 'text', maxlength: 65535, nullable: true}
     }
 };

package/core/server/models/base/plugins/actions.js

@@ -93,7 +93,7 @@ module.exports = function (Bookshelf) {
 
             const insert = (action) => {
                 Bookshelf.model('Action')
-                    .add(action)
+                    .add(action, {autoRefresh: false})
                     .catch((err) => {
                         if (_.isArray(err)) {
                             err = err[0];

package/core/server/models/base/plugins/sanitize.js

@@ -38,6 +38,8 @@ module.exports = function (Bookshelf) {
                 return baseOptions.concat('shallow', 'columns', 'previous');
             case 'destroy':
                 return baseOptions.concat(extraOptions, ['id', 'destroyBy', 'require']);
+            case 'add':
+                return baseOptions.concat(extraOptions, ['autoRefresh']);
             case 'edit':
                 return baseOptions.concat(extraOptions, ['id', 'require']);
             case 'findOne':

package/core/server/models/member.js

@@ -219,8 +219,8 @@ const Member = ghostBookshelf.Model.extend({
 
     async updateTierExpiry(products = [], options = {}) {
         for (const product of products) {
-            if (product?.expiry_at) {
-                const expiry = new Date(product.expiry_at);
+            if (product?.id) {
+                const expiry = product.expiry_at ? new Date(product.expiry_at) : null;
                 const queryOptions = _.extend({}, options, {
                     query: {where: {product_id: product.id}}
                 });

package/core/server/models/mention.js

@@ -0,0 +1,9 @@
+const ghostBookshelf = require('./base');
+
+const Mention = ghostBookshelf.Model.extend({
+    tableName: 'mentions'
+});
+
+module.exports = {
+    Mention: ghostBookshelf.model('Mention', Mention)
+};

package/core/server/models/redirect.js

@@ -36,7 +36,8 @@ const Redirect = ghostBookshelf.Model.extend({
     permittedOptions(methodName) {
         let options = ghostBookshelf.Model.permittedOptions.call(this, methodName);
         const validOptions = {
-            findAll: ['filter', 'columns', 'withRelated']
+            findAll: ['filter', 'columns', 'withRelated'],
+            edit: ['importing']
         };
 
         if (validOptions[methodName]) {

package/core/server/services/api-version-compatibility/index.js

@@ -30,21 +30,18 @@ module.exports.errorHandler = (err, req, res, next) => {
 };
 
 /**
- * If Accept-Version is set on the request set Content-Version on the response
- * Also, add 'Accept-Version' to VARY as it effects response caching
+ * Set Content-Version on the response, and add 'Accept-Version' to VARY as
+ * it effects response caching
  * TODO: move the method to mw once back-compatibility with 4.x is sorted
- * 
+ *
  * @param {import('express').Request} req
  * @param {import('express').Response} res
  * @param {import('express').NextFunction} next
  */
 module.exports.contentVersion = (req, res, next) => {
-    if (req.header('accept-version')) {
-        res.header('Content-Version', `v${ghostVersion.safe}`);
-    }
-
+    res.header('Content-Version', `v${ghostVersion.safe}`);
     res.vary('Accept-Version');
-    
+
     next();
 };
 

package/core/server/services/bulk-email/bulk-email-processor.js

@@ -93,7 +93,7 @@ module.exports = {
             } catch (error) {
                 return new FailedBatch(emailBatchId, error);
             }
-        }, {concurrency: 10});
+        }, {concurrency: 2});
 
         const successes = batchResults.filter(response => (response instanceof SuccessfulBatch));
         const failures = batchResults.filter(response => (response instanceof FailedBatch));
@@ -196,8 +196,11 @@ module.exports = {
 
             // log any error that didn't come from the provider which would have already logged it
             if (!error.code || error.code !== 'BULK_EMAIL_SEND_FAILED') {
-                let ghostError = new errors.InternalServerError({
-                    err: error
+                let ghostError = new errors.EmailError({
+                    err: error,
+                    code: 'BULK_EMAIL_SEND_FAILED',
+                    message: `Error sending email batch ${emailBatchId}`,
+                    context: error.message
                 });
                 sentry.captureException(ghostError);
                 logging.error(ghostError);
@@ -274,7 +277,7 @@ module.exports = {
             });
 
             sentry.captureException(ghostError);
-            logging.warn(ghostError);
+            logging.error(ghostError);
 
             debug(`failed to send message (${Date.now() - startTime}ms)`);
             throw ghostError;

package/core/server/services/email-service/wrapper.js

@@ -88,7 +88,8 @@ class EmailServiceWrapper {
             jobsService,
             emailSegmenter,
             emailRenderer,
-            db
+            db,
+            sentry
         });
 
         this.service = new EmailService({

package/core/server/services/email-suppression-list/MailgunEmailSuppressionList.js

@@ -73,7 +73,7 @@ class MailgunEmailSuppressionList extends AbstractEmailSuppressionList {
 
         try {
             const collection = await this.Suppression.findAll({
-                filter: `email:[${emails.join(',')}]`
+                filter: `email:[${emails.map(email => `'${email}'`).join(',')}]`
             });
 
             return emails.map((email) => {

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -37,7 +37,8 @@ module.exports = class LinkRedirectRepository {
 
     fromModel(model) {
         // Store if link has been edited
-        const edited = model.get('created_at')?.getTime() !== model.get('updated_at')?.getTime();
+        // Note: in some edge cases updated_at is set directly after created_at, sometimes with a second difference, so we need to check for that
+        const edited = model.get('updated_at')?.getTime() > (model.get('created_at')?.getTime() + 1000);
 
         return new LinkRedirect({
             id: model.id,

package/core/server/services/link-tracking/PostLinkRepository.js

@@ -80,7 +80,8 @@ module.exports = class PostLinkRepository {
         await this.#LinkRedirect.edit({
             post_id: postLink.post_id.toHexString()
         }, {
-            id: postLink.link_id.toHexString()
+            id: postLink.link_id.toHexString(),
+            importing: true // skip setting updated_at when linking a post to a link
         });
     }
 };

package/core/server/services/mega/post-email-serializer.js

@@ -400,13 +400,13 @@ const PostEmailSerializer = {
 
                 if (isSite) {
                     // Add newsletter name as ref to the URL
-                    url = memberAttribution.service.addEmailSourceAttributionTracking(url, newsletter);
+                    url = memberAttribution.service.addOutboundLinkTagging(url, newsletter);
 
                     // Only add post attribution to our own site (because external sites could/should not process this information)
                     url = memberAttribution.service.addPostAttributionTracking(url, post);
                 } else {
                     // Add email source attribution without the newsletter name
-                    url = memberAttribution.service.addEmailSourceAttributionTracking(url);
+                    url = memberAttribution.service.addOutboundLinkTagging(url);
                 }
 
                 // Add link click tracking

package/core/server/services/member-attribution/index.js

@@ -1,6 +1,7 @@
 const urlService = require('../url');
 const urlUtils = require('../../../shared/url-utils');
 const settingsCache = require('../../../shared/settings-cache');
+const labs = require('../../../shared/labs');
 
 class MemberAttributionServiceWrapper {
     init() {
@@ -41,6 +42,7 @@ class MemberAttributionServiceWrapper {
             },
             attributionBuilder: this.attributionBuilder,
             getTrackingEnabled: () => !!settingsCache.get('members_track_sources'),
+            getOutboundLinkTaggingEnabled: () => !labs.isSet('outboundLinkTagging') || !!settingsCache.get('outbound_link_tagging'),
             getSiteTitle: () => settingsCache.get('title')
         });
     }

package/core/server/services/mentions/BookshelfMentionRepository.js

@@ -0,0 +1,117 @@
+const {Mention} = require('@tryghost/webmentions');
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').IMentionRepository} IMentionRepository
+ */
+
+/**
+ * @template Model
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page<Model>} Page
+ */
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').GetPageOptions} GetPageOptions
+ */
+
+/**
+ * @implements {IMentionRepository}
+ */
+module.exports = class BookshelfMentionRepository {
+    /** @type {Object} */
+    #MentionModel;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MentionModel Bookshelf Model
+     */
+    constructor(deps) {
+        this.#MentionModel = deps.MentionModel;
+    }
+
+    #modelToMention(model) {
+        let payload;
+        try {
+            payload = JSON.parse(model.get('payload'));
+        } catch (err) {
+            logging.error(err);
+            payload = {};
+        }
+        return Mention.create({
+            id: model.get('id'),
+            source: model.get('source'),
+            target: model.get('target'),
+            timestamp: model.get('created_at'),
+            payload,
+            resourceId: model.get('resource_id'),
+            sourceTitle: model.get('source_title'),
+            sourceSiteTitle: model.get('source_site_title'),
+            sourceAuthor: model.get('source_author'),
+            sourceExcerpt: model.get('source_excerpt'),
+            sourceFavicon: model.get('source_favicon'),
+            sourceFeaturedImaged: model.get('source_featured_image')
+        });
+    }
+
+    /**
+     * @param {GetPageOptions} options
+     * @returns {Promise<Page<import('@tryghost/webmentions/lib/Mention')>>}
+     */
+    async getPage(options) {
+        const page = await this.#MentionModel.findPage(options);
+
+        return {
+            data: await Promise.all(page.data.map(model => this.#modelToMention(model))),
+            meta: page.meta
+        };
+    }
+
+    /**
+     * @param {URL} source
+     * @param {URL} target
+     * @returns {Promise<import('@tryghost/webmentions/lib/Mention')|null>}
+     */
+    async getBySourceAndTarget(source, target) {
+        const model = await this.#MentionModel.findOne({
+            source: source.href,
+            target: target.href
+        }, {require: false});
+
+        if (!model) {
+            return null;
+        }
+
+        return this.#modelToMention(model);
+    }
+
+    /**
+     * @param {import('@tryghost/webmentions/lib/Mention')} mention
+     * @returns {Promise<void>}
+     */
+    async save(mention) {
+        const data = {
+            id: mention.id.toHexString(),
+            source: mention.source.href,
+            source_title: mention.sourceTitle,
+            source_site_title: mention.sourceSiteTitle,
+            source_excerpt: mention.sourceExcerpt,
+            source_author: mention.sourceAuthor,
+            source_featured_image: mention.sourceFeaturedImage?.href,
+            source_favicon: mention.sourceFavicon?.href,
+            target: mention.target.href,
+            resource_id: mention.resourceId?.toHexString(),
+            resource_type: mention.resourceId ? 'post' : null,
+            payload: mention.payload ? JSON.stringify(mention.payload) : null
+        };
+
+        const existing = await this.#MentionModel.findOne({id: data.id}, {require: false});
+
+        if (!existing) {
+            await this.#MentionModel.add(data);
+        } else {
+            await this.#MentionModel.edit(data, {
+                id: data.id
+            });
+        }
+    }
+};

package/core/server/services/mentions/MentionController.js

@@ -0,0 +1,66 @@
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/webmentions').MentionsAPI} MentionsAPI
+ * @typedef {import('@tryghost/webmentions/lib/webmentions').Mention} Mention
+ */
+
+/**
+ * @template Model
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page} Page<Model>
+ */
+
+module.exports = class MentionController {
+    /** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
+    #api;
+
+    async init(deps) {
+        this.#api = deps.api;
+    }
+
+    /**
+     * @param {import('@tryghost/api-framework').Frame} frame
+     * @returns {Promise<Page<Mention>>}
+     */
+    async browse(frame) {
+        let limit;
+        if (!frame.options.limit || frame.options.limit === 'all') {
+            limit = 'all';
+        } else {
+            limit = parseInt(frame.options.limit);
+        }
+
+        let page;
+        if (frame.options.page) {
+            page = parseInt(frame.options.page);
+        } else {
+            page = 1;
+        }
+
+        const results = await this.#api.listMentions({
+            filter: frame.options.filter,
+            limit,
+            page
+        });
+
+        return results;
+    }
+
+    /**
+     * @param {import('@tryghost/api-framework').Frame} frame
+     * @returns {Promise<void>}
+     */
+    async receive(frame) {
+        logging.info('[Webmention] ' + JSON.stringify(frame.data));
+        const {source, target, ...payload} = frame.data;
+        const result = this.#api.processWebmention({
+            source: new URL(source),
+            target: new URL(target),
+            payload
+        });
+
+        result.catch(function rejected(err) {
+            logging.error(err);
+        });
+    }
+};

package/core/server/services/mentions/WebmentionMetadata.js

@@ -0,0 +1,20 @@
+const oembedService = require('../oembed');
+
+module.exports = class WebmentionMetadata {
+    /**
+     * @param {URL} url
+     * @returns {Promise<import('@tryghost/webmentions/lib/MentionsAPI').WebmentionMetadata>}
+     */
+    async fetch(url) {
+        const data = await oembedService.fetchOembedDataFromUrl(url.href, 'bookmark');
+        const result = {
+            siteTitle: data.metadata.publisher,
+            title: data.metadata.title,
+            excerpt: data.metadata.description,
+            author: data.metadata.author,
+            image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
+            favicon: data.metadata.icon ? new URL(data.metadata.icon) : null
+        };
+        return result;
+    }
+};

package/core/server/services/mentions/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/mentions/service.js

@@ -0,0 +1,77 @@
+const ObjectID = require('bson-objectid').default;
+const MentionController = require('./MentionController');
+const WebmentionMetadata = require('./WebmentionMetadata');
+const {
+    MentionsAPI,
+    MentionSendingService,
+    MentionDiscoveryService
+} = require('@tryghost/webmentions');
+const BookshelfMentionRepository = require('./BookshelfMentionRepository');
+const models = require('../../models');
+const events = require('../../lib/common/events');
+const externalRequest = require('../../../server/lib/request-external.js');
+const urlUtils = require('../../../shared/url-utils');
+const outputSerializerUrlUtil = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
+const labs = require('../../../shared/labs');
+const urlService = require('../url');
+
+function getPostUrl(post) {
+    const jsonModel = {};
+    outputSerializerUrlUtil.forPost(post.id, jsonModel, {options: {}});
+    return jsonModel.url;
+}
+module.exports = {
+    controller: new MentionController(),
+    async init() {
+        const repository = new BookshelfMentionRepository({
+            MentionModel: models.Mention
+        });
+        const webmentionMetadata = new WebmentionMetadata();
+        const discoveryService = new MentionDiscoveryService({externalRequest});
+        const api = new MentionsAPI({
+            repository,
+            webmentionMetadata,
+            resourceService: {
+                async getByURL(url) {
+                    const path = urlUtils.absoluteToRelative(url.href, {withoutSubdirectory: true});
+                    const resource = urlService.getResource(path);
+                    if (resource?.config?.type === 'posts') {
+                        return {
+                            type: 'post',
+                            id: ObjectID.createFromHexString(resource.data.id)
+                        };
+                    }
+                    return {
+                        type: null,
+                        id: null
+                    };
+                }
+            },
+            routingService: {
+                async pageExists(url) {
+                    const siteUrl = new URL(urlUtils.getSiteUrl());
+                    if (siteUrl.origin !== url.origin) {
+                        return false;
+                    }
+                    const subdir = urlUtils.getSubdir();
+                    if (subdir && !url.pathname.startsWith(subdir)) {
+                        return false;
+                    }
+
+                    return true;
+                }
+            }
+        });
+
+        this.controller.init({api});
+
+        const sendingService = new MentionSendingService({
+            discoveryService,
+            externalRequest,
+            getSiteUrl: () => urlUtils.urlFor('home', true),
+            getPostUrl: post => getPostUrl(post),
+            isEnabled: () => labs.isSet('webmentions')
+        });
+        sendingService.listen(events);
+    }
+};

package/core/server/services/oembed/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/oembed/service.js

@@ -0,0 +1,24 @@
+const config = require('../../../shared/config');
+const externalRequest = require('../../lib/request-external');
+
+const OEmbed = require('@tryghost/oembed-service');
+const oembed = new OEmbed({config, externalRequest});
+
+const NFT = require('./nft-oembed');
+const nft = new NFT({
+    config: {
+        apiKey: config.get('opensea').privateReadOnlyApiKey
+    }
+});
+
+const Twitter = require('./twitter-embed');
+const twitter = new Twitter({
+    config: {
+        bearerToken: config.get('twitter').privateReadOnlyToken
+    }
+});
+
+oembed.registerProvider(nft);
+oembed.registerProvider(twitter);
+
+module.exports = oembed;