ghost 5.30.0 → 5.31.0

package/core/server/data/migrations/versions/5.31/2023-01-17-14-59-add-outbound-link-tagging-setting.js

@@ -0,0 +1,8 @@
+const {addSetting} = require('../../utils');
+
+module.exports = addSetting({
+    key: 'outbound_link_tagging',
+    value: 'true',
+    type: 'boolean',
+    group: 'analytics'
+});

package/core/server/data/migrations/versions/5.31/2023-01-19-07-46-add-mentions-table.js

@@ -0,0 +1,17 @@
+const {addTable} = require('../../utils');
+
+module.exports = addTable('mentions', {
+    id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+    source: {type: 'string', maxlength: 2000, nullable: false},
+    source_title: {type: 'string', maxlength: 2000, nullable: true},
+    source_site_title: {type: 'string', maxlength: 2000, nullable: true},
+    source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
+    source_author: {type: 'string', maxlength: 2000, nullable: true},
+    source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
+    source_favicon: {type: 'string', maxlength: 2000, nullable: true},
+    target: {type: 'string', maxlength: 2000, nullable: false},
+    resource_id: {type: 'string', maxlength: 24, nullable: true},
+    resource_type: {type: 'string', maxlength: 50, nullable: true},
+    created_at: {type: 'dateTime', nullable: false},
+    payload: {type: 'text', maxlength: 65535, nullable: true}
+});

package/core/server/data/schema/default-settings/default-settings.json

@@ -480,5 +480,15 @@
                 ]]
             }
         }
+    },
+    "analytics": {
+        "outbound_link_tagging": {
+            "defaultValue": "true",
+            "validations": {
+                "isEmpty": false,
+                "isIn": [["true", "false"]]
+            },
+            "type": "boolean"
+        }
     }
 }

package/core/server/data/schema/schema.js

@@ -979,5 +979,20 @@ module.exports = {
         '@@UNIQUE_CONSTRAINTS@@': [
             ['email_id', 'member_id']
         ]
+    },
+    mentions: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        source: {type: 'string', maxlength: 2000, nullable: false},
+        source_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_site_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
+        source_author: {type: 'string', maxlength: 2000, nullable: true},
+        source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
+        source_favicon: {type: 'string', maxlength: 2000, nullable: true},
+        target: {type: 'string', maxlength: 2000, nullable: false},
+        resource_id: {type: 'string', maxlength: 24, nullable: true},
+        resource_type: {type: 'string', maxlength: 50, nullable: true},
+        created_at: {type: 'dateTime', nullable: false},
+        payload: {type: 'text', maxlength: 65535, nullable: true}
     }
 };

package/core/server/models/base/plugins/actions.js

@@ -93,7 +93,7 @@ module.exports = function (Bookshelf) {
 
             const insert = (action) => {
                 Bookshelf.model('Action')
-                    .add(action)
+                    .add(action, {autoRefresh: false})
                     .catch((err) => {
                         if (_.isArray(err)) {
                             err = err[0];

package/core/server/models/base/plugins/sanitize.js

@@ -38,6 +38,8 @@ module.exports = function (Bookshelf) {
                 return baseOptions.concat('shallow', 'columns', 'previous');
             case 'destroy':
                 return baseOptions.concat(extraOptions, ['id', 'destroyBy', 'require']);
+            case 'add':
+                return baseOptions.concat(extraOptions, ['autoRefresh']);
             case 'edit':
                 return baseOptions.concat(extraOptions, ['id', 'require']);
             case 'findOne':

package/core/server/models/mention.js

@@ -0,0 +1,9 @@
+const ghostBookshelf = require('./base');
+
+const Mention = ghostBookshelf.Model.extend({
+    tableName: 'mentions'
+});
+
+module.exports = {
+    Mention: ghostBookshelf.model('Mention', Mention)
+};

package/core/server/models/redirect.js

@@ -36,7 +36,8 @@ const Redirect = ghostBookshelf.Model.extend({
     permittedOptions(methodName) {
         let options = ghostBookshelf.Model.permittedOptions.call(this, methodName);
         const validOptions = {
-            findAll: ['filter', 'columns', 'withRelated']
+            findAll: ['filter', 'columns', 'withRelated'],
+            edit: ['importing']
         };
 
         if (validOptions[methodName]) {

package/core/server/services/api-version-compatibility/index.js

@@ -30,21 +30,18 @@ module.exports.errorHandler = (err, req, res, next) => {
 };
 
 /**
- * If Accept-Version is set on the request set Content-Version on the response
- * Also, add 'Accept-Version' to VARY as it effects response caching
+ * Set Content-Version on the response, and add 'Accept-Version' to VARY as
+ * it effects response caching
  * TODO: move the method to mw once back-compatibility with 4.x is sorted
- * 
+ *
  * @param {import('express').Request} req
  * @param {import('express').Response} res
  * @param {import('express').NextFunction} next
  */
 module.exports.contentVersion = (req, res, next) => {
-    if (req.header('accept-version')) {
-        res.header('Content-Version', `v${ghostVersion.safe}`);
-    }
-
+    res.header('Content-Version', `v${ghostVersion.safe}`);
     res.vary('Accept-Version');
-    
+
     next();
 };
 

package/core/server/services/email-suppression-list/MailgunEmailSuppressionList.js

@@ -73,7 +73,7 @@ class MailgunEmailSuppressionList extends AbstractEmailSuppressionList {
 
         try {
             const collection = await this.Suppression.findAll({
-                filter: `email:[${emails.join(',')}]`
+                filter: `email:[${emails.map(email => `'${email}'`).join(',')}]`
             });
 
             return emails.map((email) => {

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -37,7 +37,8 @@ module.exports = class LinkRedirectRepository {
 
     fromModel(model) {
         // Store if link has been edited
-        const edited = model.get('created_at')?.getTime() !== model.get('updated_at')?.getTime();
+        // Note: in some edge cases updated_at is set directly after created_at, sometimes with a second difference, so we need to check for that
+        const edited = model.get('updated_at')?.getTime() > (model.get('created_at')?.getTime() + 1000);
 
         return new LinkRedirect({
             id: model.id,

package/core/server/services/link-tracking/PostLinkRepository.js

@@ -80,7 +80,8 @@ module.exports = class PostLinkRepository {
         await this.#LinkRedirect.edit({
             post_id: postLink.post_id.toHexString()
         }, {
-            id: postLink.link_id.toHexString()
+            id: postLink.link_id.toHexString(),
+            importing: true // skip setting updated_at when linking a post to a link
         });
     }
 };

package/core/server/services/mega/post-email-serializer.js

@@ -400,13 +400,13 @@ const PostEmailSerializer = {
 
                 if (isSite) {
                     // Add newsletter name as ref to the URL
-                    url = memberAttribution.service.addEmailSourceAttributionTracking(url, newsletter);
+                    url = memberAttribution.service.addOutboundLinkTagging(url, newsletter);
 
                     // Only add post attribution to our own site (because external sites could/should not process this information)
                     url = memberAttribution.service.addPostAttributionTracking(url, post);
                 } else {
                     // Add email source attribution without the newsletter name
-                    url = memberAttribution.service.addEmailSourceAttributionTracking(url);
+                    url = memberAttribution.service.addOutboundLinkTagging(url);
                 }
 
                 // Add link click tracking

package/core/server/services/member-attribution/index.js

@@ -1,6 +1,7 @@
 const urlService = require('../url');
 const urlUtils = require('../../../shared/url-utils');
 const settingsCache = require('../../../shared/settings-cache');
+const labs = require('../../../shared/labs');
 
 class MemberAttributionServiceWrapper {
     init() {
@@ -41,6 +42,7 @@ class MemberAttributionServiceWrapper {
             },
             attributionBuilder: this.attributionBuilder,
             getTrackingEnabled: () => !!settingsCache.get('members_track_sources'),
+            getOutboundLinkTaggingEnabled: () => !labs.isSet('outboundLinkTagging') || !!settingsCache.get('outbound_link_tagging'),
             getSiteTitle: () => settingsCache.get('title')
         });
     }

package/core/server/services/mentions/BookshelfMentionRepository.js

@@ -0,0 +1,117 @@
+const {Mention} = require('@tryghost/webmentions');
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').IMentionRepository} IMentionRepository
+ */
+
+/**
+ * @template Model
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page<Model>} Page
+ */
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').GetPageOptions} GetPageOptions
+ */
+
+/**
+ * @implements {IMentionRepository}
+ */
+module.exports = class BookshelfMentionRepository {
+    /** @type {Object} */
+    #MentionModel;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MentionModel Bookshelf Model
+     */
+    constructor(deps) {
+        this.#MentionModel = deps.MentionModel;
+    }
+
+    #modelToMention(model) {
+        let payload;
+        try {
+            payload = JSON.parse(model.get('payload'));
+        } catch (err) {
+            logging.error(err);
+            payload = {};
+        }
+        return Mention.create({
+            id: model.get('id'),
+            source: model.get('source'),
+            target: model.get('target'),
+            timestamp: model.get('created_at'),
+            payload,
+            resourceId: model.get('resource_id'),
+            sourceTitle: model.get('source_title'),
+            sourceSiteTitle: model.get('source_site_title'),
+            sourceAuthor: model.get('source_author'),
+            sourceExcerpt: model.get('source_excerpt'),
+            sourceFavicon: model.get('source_favicon'),
+            sourceFeaturedImaged: model.get('source_featured_image')
+        });
+    }
+
+    /**
+     * @param {GetPageOptions} options
+     * @returns {Promise<Page<import('@tryghost/webmentions/lib/Mention')>>}
+     */
+    async getPage(options) {
+        const page = await this.#MentionModel.findPage(options);
+
+        return {
+            data: await Promise.all(page.data.map(model => this.#modelToMention(model))),
+            meta: page.meta
+        };
+    }
+
+    /**
+     * @param {URL} source
+     * @param {URL} target
+     * @returns {Promise<import('@tryghost/webmentions/lib/Mention')|null>}
+     */
+    async getBySourceAndTarget(source, target) {
+        const model = await this.#MentionModel.findOne({
+            source: source.href,
+            target: target.href
+        }, {require: false});
+
+        if (!model) {
+            return null;
+        }
+
+        return this.#modelToMention(model);
+    }
+
+    /**
+     * @param {import('@tryghost/webmentions/lib/Mention')} mention
+     * @returns {Promise<void>}
+     */
+    async save(mention) {
+        const data = {
+            id: mention.id.toHexString(),
+            source: mention.source.href,
+            source_title: mention.sourceTitle,
+            source_site_title: mention.sourceSiteTitle,
+            source_excerpt: mention.sourceExcerpt,
+            source_author: mention.sourceAuthor,
+            source_featured_image: mention.sourceFeaturedImage?.href,
+            source_favicon: mention.sourceFavicon?.href,
+            target: mention.target.href,
+            resource_id: mention.resourceId?.toHexString(),
+            resource_type: mention.resourceId ? 'post' : null,
+            payload: mention.payload ? JSON.stringify(mention.payload) : null
+        };
+
+        const existing = await this.#MentionModel.findOne({id: data.id}, {require: false});
+
+        if (!existing) {
+            await this.#MentionModel.add(data);
+        } else {
+            await this.#MentionModel.edit(data, {
+                id: data.id
+            });
+        }
+    }
+};

package/core/server/services/mentions/MentionController.js

@@ -0,0 +1,66 @@
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('@tryghost/webmentions/lib/webmentions').MentionsAPI} MentionsAPI
+ * @typedef {import('@tryghost/webmentions/lib/webmentions').Mention} Mention
+ */
+
+/**
+ * @template Model
+ * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page} Page<Model>
+ */
+
+module.exports = class MentionController {
+    /** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
+    #api;
+
+    async init(deps) {
+        this.#api = deps.api;
+    }
+
+    /**
+     * @param {import('@tryghost/api-framework').Frame} frame
+     * @returns {Promise<Page<Mention>>}
+     */
+    async browse(frame) {
+        let limit;
+        if (!frame.options.limit || frame.options.limit === 'all') {
+            limit = 'all';
+        } else {
+            limit = parseInt(frame.options.limit);
+        }
+
+        let page;
+        if (frame.options.page) {
+            page = parseInt(frame.options.page);
+        } else {
+            page = 1;
+        }
+
+        const results = await this.#api.listMentions({
+            filter: frame.options.filter,
+            limit,
+            page
+        });
+
+        return results;
+    }
+
+    /**
+     * @param {import('@tryghost/api-framework').Frame} frame
+     * @returns {Promise<void>}
+     */
+    async receive(frame) {
+        logging.info('[Webmention] ' + JSON.stringify(frame.data));
+        const {source, target, ...payload} = frame.data;
+        const result = this.#api.processWebmention({
+            source: new URL(source),
+            target: new URL(target),
+            payload
+        });
+
+        result.catch(function rejected(err) {
+            logging.error(err);
+        });
+    }
+};

package/core/server/services/mentions/WebmentionMetadata.js

@@ -0,0 +1,20 @@
+const oembedService = require('../oembed');
+
+module.exports = class WebmentionMetadata {
+    /**
+     * @param {URL} url
+     * @returns {Promise<import('@tryghost/webmentions/lib/MentionsAPI').WebmentionMetadata>}
+     */
+    async fetch(url) {
+        const data = await oembedService.fetchOembedDataFromUrl(url.href, 'bookmark');
+        const result = {
+            siteTitle: data.metadata.publisher,
+            title: data.metadata.title,
+            excerpt: data.metadata.description,
+            author: data.metadata.author,
+            image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
+            favicon: data.metadata.icon ? new URL(data.metadata.icon) : null
+        };
+        return result;
+    }
+};

package/core/server/services/mentions/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/mentions/service.js

@@ -0,0 +1,77 @@
+const ObjectID = require('bson-objectid').default;
+const MentionController = require('./MentionController');
+const WebmentionMetadata = require('./WebmentionMetadata');
+const {
+    MentionsAPI,
+    MentionSendingService,
+    MentionDiscoveryService
+} = require('@tryghost/webmentions');
+const BookshelfMentionRepository = require('./BookshelfMentionRepository');
+const models = require('../../models');
+const events = require('../../lib/common/events');
+const externalRequest = require('../../../server/lib/request-external.js');
+const urlUtils = require('../../../shared/url-utils');
+const outputSerializerUrlUtil = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
+const labs = require('../../../shared/labs');
+const urlService = require('../url');
+
+function getPostUrl(post) {
+    const jsonModel = {};
+    outputSerializerUrlUtil.forPost(post.id, jsonModel, {options: {}});
+    return jsonModel.url;
+}
+module.exports = {
+    controller: new MentionController(),
+    async init() {
+        const repository = new BookshelfMentionRepository({
+            MentionModel: models.Mention
+        });
+        const webmentionMetadata = new WebmentionMetadata();
+        const discoveryService = new MentionDiscoveryService({externalRequest});
+        const api = new MentionsAPI({
+            repository,
+            webmentionMetadata,
+            resourceService: {
+                async getByURL(url) {
+                    const path = urlUtils.absoluteToRelative(url.href, {withoutSubdirectory: true});
+                    const resource = urlService.getResource(path);
+                    if (resource?.config?.type === 'posts') {
+                        return {
+                            type: 'post',
+                            id: ObjectID.createFromHexString(resource.data.id)
+                        };
+                    }
+                    return {
+                        type: null,
+                        id: null
+                    };
+                }
+            },
+            routingService: {
+                async pageExists(url) {
+                    const siteUrl = new URL(urlUtils.getSiteUrl());
+                    if (siteUrl.origin !== url.origin) {
+                        return false;
+                    }
+                    const subdir = urlUtils.getSubdir();
+                    if (subdir && !url.pathname.startsWith(subdir)) {
+                        return false;
+                    }
+
+                    return true;
+                }
+            }
+        });
+
+        this.controller.init({api});
+
+        const sendingService = new MentionSendingService({
+            discoveryService,
+            externalRequest,
+            getSiteUrl: () => urlUtils.urlFor('home', true),
+            getPostUrl: post => getPostUrl(post),
+            isEnabled: () => labs.isSet('webmentions')
+        });
+        sendingService.listen(events);
+    }
+};

package/core/server/services/oembed/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/oembed/service.js

@@ -0,0 +1,24 @@
+const config = require('../../../shared/config');
+const externalRequest = require('../../lib/request-external');
+
+const OEmbed = require('@tryghost/oembed-service');
+const oembed = new OEmbed({config, externalRequest});
+
+const NFT = require('./nft-oembed');
+const nft = new NFT({
+    config: {
+        apiKey: config.get('opensea').privateReadOnlyApiKey
+    }
+});
+
+const Twitter = require('./twitter-embed');
+const twitter = new Twitter({
+    config: {
+        bearerToken: config.get('twitter').privateReadOnlyToken
+    }
+});
+
+oembed.registerProvider(nft);
+oembed.registerProvider(twitter);
+
+module.exports = oembed;

package/core/server/services/url/Resources.js

@@ -21,10 +21,11 @@ class Resources {
      * @param {Object} options
      * @param {Object} [options.resources] - resources to initialize with instead of fetching them from the database
      * @param {Object} [options.queue] - instance of the Queue class
+     * @param {Object[]} [options.resourcesConfig] - resource config used when handling resource events and fetching
      */
-    constructor({resources = {}, queue} = {}) {
+    constructor({resources = {}, queue, resourcesConfig = []} = {}) {
         this.queue = queue;
-        this.resourcesConfig = [];
+        this.resourcesConfig = resourcesConfig;
         this.data = resources;
 
         this.listeners = [];
@@ -47,20 +48,6 @@ class Resources {
         events.on(eventName, listener);
     }
 
-    /**
-     * @description Initialize the resource config. We currently fetch the data straight via the the model layer,
-     *              but because Ghost supports multiple API versions, we have to ensure we load the correct data.
-     *
-     * @TODO: https://github.com/TryGhost/Ghost/issues/10360
-     */
-    initResourceConfig() {
-        if (!_.isEmpty(this.resourcesConfig)) {
-            return;
-        }
-
-        this.resourcesConfig = require('./config');
-    }
-
     /**
      * @description Helper function to initialize data fetching.
      */
@@ -440,7 +427,6 @@ class Resources {
 
         this.listeners = [];
         this.data = {};
-        this.resourcesConfig = null;
     }
 
     /**

package/core/server/services/url/UrlService.js

@@ -8,6 +8,7 @@ const Queue = require('./Queue');
 const Urls = require('./Urls');
 const Resources = require('./Resources');
 const urlUtils = require('../../../shared/url-utils');
+const resourcesConfig = require('./config');
 
 /**
  * The url service class holds all instances in a centralized place.
@@ -17,7 +18,7 @@ const urlUtils = require('../../../shared/url-utils');
 class UrlService {
     /**
      *
-     * @param {Object} options
+     * @param {Object} [options]
      * @param {Object} [options.cache] - cache handler instance
      * @param {Function} [options.cache.read] - read cache by type
      * @param {Function} [options.cache.write] - write into cache by type
@@ -35,6 +36,7 @@ class UrlService {
         //      Way too many tests fail if the initialization is removed so leaving it as is for time being
         this.urls = new Urls();
         this.resources = new Resources({
+            resourcesConfig: resourcesConfig,
             queue: this.queue
         });
 
@@ -321,12 +323,10 @@ class UrlService {
         if (persistedUrls && persistedResources) {
             this.urls.urls = persistedUrls;
             this.resources.data = persistedResources;
-            this.resources.initResourceConfig();
             this.resources.initEventListeners();
 
             this._onQueueEnded('init');
         } else {
-            this.resources.initResourceConfig();
             this.resources.initEventListeners();
             await this.resources.fetchResources();
             // CASE: all resources are fetched, start the queue

package/core/server/web/api/endpoints/admin/routes.js

@@ -5,6 +5,7 @@ const apiMw = require('../../middleware');
 const mw = require('./middleware');
 
 const shared = require('../../../shared');
+const labs = require('../../../../../shared/labs');
 
 module.exports = function apiRoutes() {
     const router = express.Router('admin api');
@@ -31,6 +32,8 @@ module.exports = function apiRoutes() {
     router.put('/posts/:id', mw.authAdminApi, http(api.posts.edit));
     router.del('/posts/:id', mw.authAdminApi, http(api.posts.destroy));
 
+    router.get('/mentions', labs.enabledMiddleware('webmentions'), mw.authAdminApi, http(api.mentions.browse));
+
     router.put('/comments/:id', mw.authAdminApi, http(api.comments.edit));
 
     // ## Pages

package/core/server/web/parent/frontend.js

@@ -18,6 +18,7 @@ module.exports = (routerConfig) => {
     frontendApp.use(shared.middleware.urlRedirects.frontendSSLRedirect);
 
     frontendApp.lazyUse('/members', require('../members'));
+    frontendApp.lazyUse('/webmentions', require('../webmentions'));
     frontendApp.use('/', require('../../../frontend/web')(routerConfig));
 
     return frontendApp;

package/core/server/web/webmentions/index.js

@@ -0,0 +1 @@
+module.exports = require('./routes');

package/core/server/web/webmentions/routes.js

@@ -0,0 +1,18 @@
+const express = require('../../../shared/express');
+const api = require('../../api').endpoints;
+const {http} = require('@tryghost/api-framework');
+const shared = require('../shared');
+
+const bodyParser = require('body-parser');
+
+module.exports = function apiRoutes() {
+    const router = express.Router('webmentions');
+
+    // shouldn't be cached
+    router.use(shared.middleware.cacheControl('private'));
+
+    // Webmentions
+    router.post('/receive', bodyParser.urlencoded({extended: true, limit: '5mb'}), http(api.mentions.receive));
+
+    return router;
+};