ghost 5.15.0 → 5.16.0

package/core/server/api/endpoints/utils/serializers/input/posts.js

@@ -6,6 +6,7 @@ const localUtils = require('../../index');
 const mobiledoc = require('../../../../../lib/mobiledoc');
 const postsMetaSchema = require('../../../../../data/schema').tables.posts_meta;
 const clean = require('./utils/clean');
+const labs = require('../../../../../../shared/labs');
 
 function removeSourceFormats(frame) {
     if (frame.options.formats?.includes('mobiledoc') || frame.options.formats?.includes('lexical')) {
@@ -24,7 +25,11 @@ function defaultRelations(frame) {
         return false;
     }
 
-    frame.options.withRelated = ['tags', 'authors', 'authors.roles', 'email', 'tiers', 'newsletter', 'count.signups', 'count.conversions'];
+    if (labs.isSet('emailClicks')) {
+        frame.options.withRelated = ['tags', 'authors', 'authors.roles', 'email', 'tiers', 'newsletter', 'count.signups', 'count.conversions', 'count.clicks'];
+    } else {
+        frame.options.withRelated = ['tags', 'authors', 'authors.roles', 'email', 'tiers', 'newsletter', 'count.signups', 'count.conversions'];
+    }
 }
 
 function setDefaultOrder(frame) {

package/core/server/api/endpoints/utils/serializers/output/mappers/activity-feed-events.js

@@ -1,4 +1,6 @@
 const mapComment = require('./comments');
+const url = require('../utils/url');
+const _ = require('lodash');
 
 const commentEventMapper = (json, frame) => {
     return {
@@ -7,10 +9,59 @@ const commentEventMapper = (json, frame) => {
     };
 };
 
+const clickEventMapper = (json, frame) => {
+    const memberFields = [
+        'id',
+        'uuid',
+        'name',
+        'email',
+        'avatar_image'
+    ];
+
+    const linkFields = [
+        'from',
+        'to'
+    ];
+
+    const postFields = [
+        'id',
+        'uuid',
+        'title',
+        'url'
+    ];
+
+    const data = json.data;
+    const response = {};
+
+    if (data.link && data.link.post) {
+        // We could use the post mapper here, but we need less field + don't need al the async behavior support
+        url.forPost(data.link.post.id, data.link.post, frame);
+        response.post = _.pick(data.link.post, postFields);
+    }
+
+    if (data.link) {
+        response.link = _.pick(data.link, linkFields);
+    }
+
+    if (data.member) {
+        response.member = _.pick(data.member, memberFields);
+    } else {
+        response.member = null;
+    }
+
+    return {
+        ...json,
+        data: response
+    };
+};
+
 const activityFeedMapper = (event, frame) => {
     if (event.type === 'comment_event') {
         return commentEventMapper(event, frame);
     }
+    if (event.type === 'click_event') {
+        return clickEventMapper(event, frame);
+    }
     return event;
 };
 

package/core/server/api/endpoints/utils/serializers/output/mappers/comments.js

@@ -18,6 +18,15 @@ const memberFields = [
     'avatar_image'
 ];
 
+const memberFieldsAdmin = [
+    'id',
+    'uuid',
+    'name',
+    'email',
+    'expertise',
+    'avatar_image'
+];
+
 const postFields = [
     'id',
     'uuid',
@@ -36,7 +45,7 @@ const commentMapper = (model, frame) => {
     const response = _.pick(jsonModel, commentFields);
 
     if (jsonModel.member) {
-        response.member = _.pick(jsonModel.member, memberFields);
+        response.member = _.pick(jsonModel.member, utils.isMembersAPI(frame) ? memberFields : memberFieldsAdmin);
     } else {
         response.member = null;
     }

package/core/server/api/endpoints/utils/serializers/output/mappers/posts.js

@@ -110,7 +110,7 @@ module.exports = async (model, frame, options = {}) => {
         });
     }
 
-    if (!labs.isSet('memberAttribution')) {
+    if (!labs.isSet('memberAttribution') && !labs.isSet('emailClicks')) {
         delete jsonModel.count;
     }
 

package/core/server/data/exporter/table-lists.js

@@ -38,7 +38,9 @@ const BACKUP_TABLES = [
     'comments',
     'comment_likes',
     'comment_reports',
-    'jobs'
+    'jobs',
+    'link_redirects',
+    'members_link_click_events'
 ];
 
 // NOTE: exposing only tables which are going to be included in a "default" export file

package/core/server/data/migrations/versions/5.16/2022-09-19-09-04-add-link-redirects-table.js

@@ -0,0 +1,10 @@
+const {addTable} = require('../../utils');
+
+module.exports = addTable('link_redirects', {
+    id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+    from: {type: 'string', maxlength: 2000, nullable: false},
+    to: {type: 'string', maxlength: 2000, nullable: false},
+    post_id: {type: 'string', maxlength: 24, nullable: true, unique: false, references: 'posts.id', setNullDelete: true},
+    created_at: {type: 'dateTime', nullable: false},
+    updated_at: {type: 'dateTime', nullable: true}
+});

package/core/server/data/migrations/versions/5.16/2022-09-19-09-05-add-members-link-click-events-table.js

@@ -0,0 +1,8 @@
+const {addTable} = require('../../utils');
+
+module.exports = addTable('members_link_click_events', {
+    id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+    member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
+    link_id: {type: 'string', maxlength: 24, nullable: false, references: 'link_redirects.id', cascadeDelete: true},
+    created_at: {type: 'dateTime', nullable: false}
+});

package/core/server/data/migrations/versions/5.16/2022-09-19-17-44-add-referrer-columns-to-member-events-table.js

@@ -0,0 +1,21 @@
+const {createAddColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createAddColumnMigration('members_created_events', 'referrer_source', {
+        type: 'string',
+        maxlength: 191,
+        nullable: true
+    }),
+
+    createAddColumnMigration('members_created_events', 'referrer_medium', {
+        type: 'string',
+        maxlength: 191,
+        nullable: true
+    }),
+
+    createAddColumnMigration('members_created_events', 'referrer_url', {
+        type: 'string',
+        maxlength: 2000,
+        nullable: true
+    })
+);

package/core/server/data/migrations/versions/5.16/2022-09-19-17-44-add-referrer-columns-to-subscription-events-table.js

@@ -0,0 +1,21 @@
+const {createAddColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createAddColumnMigration('members_subscription_created_events', 'referrer_source', {
+        type: 'string',
+        maxlength: 191,
+        nullable: true
+    }),
+
+    createAddColumnMigration('members_subscription_created_events', 'referrer_medium', {
+        type: 'string',
+        maxlength: 191,
+        nullable: true
+    }),
+
+    createAddColumnMigration('members_subscription_created_events', 'referrer_url', {
+        type: 'string',
+        maxlength: 2000,
+        nullable: true
+    })
+);

package/core/server/data/schema/schema.js

@@ -497,6 +497,9 @@ module.exports = {
             }
         },
         attribution_url: {type: 'string', maxlength: 2000, nullable: true},
+        referrer_source: {type: 'string', maxlength: 191, nullable: true},
+        referrer_medium: {type: 'string', maxlength: 191, nullable: true},
+        referrer_url: {type: 'string', maxlength: 2000, nullable: true},
         source: {
             type: 'string', maxlength: 50, nullable: false, validations: {
                 isIn: [['member', 'import', 'system', 'api', 'admin']]
@@ -634,7 +637,10 @@ module.exports = {
                 isIn: [['url', 'post', 'page', 'author', 'tag']]
             }
         },
-        attribution_url: {type: 'string', maxlength: 2000, nullable: true}
+        attribution_url: {type: 'string', maxlength: 2000, nullable: true},
+        referrer_source: {type: 'string', maxlength: 191, nullable: true},
+        referrer_medium: {type: 'string', maxlength: 191, nullable: true},
+        referrer_url: {type: 'string', maxlength: 2000, nullable: true}
     },
     offer_redemptions: {
         id: {type: 'string', maxlength: 24, nullable: false, primary: true},
@@ -829,5 +835,19 @@ module.exports = {
         finished_at: {type: 'dateTime', nullable: true},
         created_at: {type: 'dateTime', nullable: false},
         updated_at: {type: 'dateTime', nullable: true}
+    },
+    link_redirects: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        from: {type: 'string', maxlength: 2000, nullable: false},
+        to: {type: 'string', maxlength: 2000, nullable: false},
+        post_id: {type: 'string', maxlength: 24, nullable: true, unique: false, references: 'posts.id', setNullDelete: true},
+        created_at: {type: 'dateTime', nullable: false},
+        updated_at: {type: 'dateTime', nullable: true}
+    },
+    members_link_click_events: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
+        link_id: {type: 'string', maxlength: 24, nullable: false, references: 'link_redirects.id', cascadeDelete: true},
+        created_at: {type: 'dateTime', nullable: false}
     }
 };

package/core/server/models/link-redirect.js

@@ -0,0 +1,65 @@
+const ghostBookshelf = require('./base');
+const urlUtils = require('../../shared/url-utils');
+
+const LinkRedirect = ghostBookshelf.Model.extend({
+    tableName: 'link_redirects',
+
+    post() {
+        return this.belongsTo('Post', 'post_id');
+    },
+
+    formatOnWrite(attrs) {
+        if (attrs.to) {
+            attrs.to = urlUtils.absoluteToTransformReady(attrs.to);
+        }
+
+        return attrs;
+    },
+
+    parse() {
+        const attrs = ghostBookshelf.Model.prototype.parse.apply(this, arguments);
+
+        if (attrs.to) {
+            attrs.to = urlUtils.transformReadyToAbsolute(attrs.to);
+        }
+
+        return attrs;
+    }
+}, {
+    orderDefaultRaw(options) {
+        if (options.withRelated && options.withRelated.includes('count.clicks')) {
+            return '`count__clicks` DESC, `to` DESC';
+        }
+        return '`to` DESC';
+    },
+
+    permittedOptions(methodName) {
+        let options = ghostBookshelf.Model.permittedOptions.call(this, methodName);
+        const validOptions = {
+            findAll: ['filter', 'columns', 'withRelated']
+        };
+
+        if (validOptions[methodName]) {
+            options = options.concat(validOptions[methodName]);
+        }
+
+        return options;
+    },
+
+    countRelations() {
+        return {
+            clicks(modelOrCollection) {
+                modelOrCollection.query('columns', 'link_redirects.*', (qb) => {
+                    qb.countDistinct('members_link_click_events.member_id')
+                        .from('members_link_click_events')
+                        .whereRaw('link_redirects.id = members_link_click_events.link_id')
+                        .as('count__clicks');
+                });
+            }
+        };
+    }
+});
+
+module.exports = {
+    LinkRedirect: ghostBookshelf.model('LinkRedirect', LinkRedirect)
+};

package/core/server/models/member-link-click-event.js

@@ -0,0 +1,26 @@
+const errors = require('@tryghost/errors');
+const ghostBookshelf = require('./base');
+
+const MemberLinkClickEvent = ghostBookshelf.Model.extend({
+    tableName: 'members_link_click_events',
+
+    link() {
+        return this.belongsTo('LinkRedirect', 'link_id');
+    },
+
+    member() {
+        return this.belongsTo('Member', 'member_id', 'id');
+    }
+}, {
+    async edit() {
+        throw new errors.IncorrectUsageError({message: 'Cannot edit MemberLinkClickEvent'});
+    },
+
+    async destroy() {
+        throw new errors.IncorrectUsageError({message: 'Cannot destroy MemberLinkClickEvent'});
+    }
+});
+
+module.exports = {
+    MemberLinkClickEvent: ghostBookshelf.model('MemberLinkClickEvent', MemberLinkClickEvent)
+};

package/core/server/models/post.js

@@ -26,7 +26,11 @@ const messages = {
     expectedPublishedAtInFuture: 'Date must be at least {cannotScheduleAPostBeforeInMinutes} minutes in the future.',
     untitled: '(Untitled)',
     notEnoughPermission: 'You do not have permission to perform this action',
-    invalidNewsletter: 'The newsletter parameter doesn\'t match any active newsletter.'
+    invalidNewsletter: 'The newsletter parameter doesn\'t match any active newsletter.',
+    invalidMobiledocStructure: 'Invalid mobiledoc structure.',
+    invalidMobiledocStructureHelp: 'https://ghost.org/docs/publishing/',
+    invalidLexicalStructure: 'Invalid lexical structure.',
+    invalidLexicalStructureHelp: 'https://ghost.org/docs/publishing/'
 };
 
 const MOBILEDOC_REVISIONS_COUNT = 10;
@@ -130,6 +134,7 @@ Post = ghostBookshelf.Model.extend({
         // transform URLs from __GHOST_URL__ to absolute
         [
             'mobiledoc',
+            'lexical',
             'html',
             'plaintext',
             'custom_excerpt',
@@ -158,6 +163,7 @@ Post = ghostBookshelf.Model.extend({
                     cardTransformers: mobiledocLib.cards
                 }
             },
+            lexical: 'lexicalToTransformReady',
             html: 'htmlToTransformReady',
             plaintext: 'plaintextToTransformReady',
             custom_excerpt: 'htmlToTransformReady',
@@ -623,7 +629,7 @@ Post = ghostBookshelf.Model.extend({
                 this.set('html', mobiledocLib.mobiledocHtmlRenderer.render(JSON.parse(this.get('mobiledoc'))));
             } catch (err) {
                 throw new errors.ValidationError({
-                    message: 'Invalid mobiledoc structure.',
+                    message: tpl(messages.invalidMobiledocStructure),
                     help: 'https://ghost.org/docs/publishing/'
                 });
             }
@@ -644,9 +650,10 @@ Post = ghostBookshelf.Model.extend({
                 this.set('html', lexicalLib.lexicalHtmlRenderer.render(this.get('lexical')));
             } catch (err) {
                 throw new errors.ValidationError({
-                    message: 'Invalid lexical structure.',
-                    help: 'https://ghost.org/docs/publishing/',
-                    property: 'lexical'
+                    message: tpl(messages.invalidLexicalStructure),
+                    context: err.message,
+                    property: 'lexical',
+                    help: tpl(messages.invalidLexicalStructureHelp)
                 });
             }
         }
@@ -1338,6 +1345,15 @@ Post = ghostBookshelf.Model.extend({
                         .whereRaw('posts.id = members_subscription_created_events.attribution_id')
                         .as('count__conversions');
                 });
+            },
+            clicks(modelOrCollection) {
+                modelOrCollection.query('columns', 'posts.*', (qb) => {
+                    qb.countDistinct('members_link_click_events.member_id')
+                        .from('members_link_click_events')
+                        .join('link_redirects', 'members_link_click_events.link_id', 'link_redirects.id')
+                        .whereRaw('posts.id = link_redirects.post_id')
+                        .as('count__clicks');
+                });
             }
         };
     }

package/core/server/services/bulk-email/bulk-email-processor.js

@@ -13,7 +13,7 @@ const configService = require('../../../shared/config');
 const settingsCache = require('../../../shared/settings-cache');
 
 const messages = {
-    error: 'The email service was unable to send an email batch.'
+    error: 'The email service received an error from mailgun and was unable to send.'
 };
 
 const mailgunClient = new MailgunClient({config: configService, settings: settingsCache});
@@ -251,11 +251,13 @@ module.exports = {
             const response = await mailgunClient.send(emailData, recipientData, replacements);
             debug(`sent message (${Date.now() - startTime}ms)`);
             return response;
-        } catch (error) {
-            // REF: possible mailgun errors https://documentation.mailgun.com/en/latest/api-intro.html#errors
+        } catch (err) {
             let ghostError = new errors.EmailError({
-                err: error,
-                context: tpl(messages.error),
+                err,
+                message: tpl(messages.error),
+                context: `Mailgun Error ${err.error.status}: ${err.error.details}`,
+                // REF: possible mailgun errors https://documentation.mailgun.com/en/latest/api-intro.html#errors
+                help: `https://ghost.org/docs/newsletters/#bulk-email-configuration`,
                 code: 'BULK_EMAIL_SEND_FAILED'
             });
 

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -0,0 +1,88 @@
+const LinkRedirect = require('@tryghost/link-redirects').LinkRedirect;
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkRedirectRepository {
+    /** @type {Object} */
+    #LinkRedirect;
+    /** @type {Object} */
+    #urlUtils;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.LinkRedirect Bookshelf Model
+     * @param {object} deps.urlUtils
+     */
+    constructor(deps) {
+        this.#LinkRedirect = deps.LinkRedirect;
+        this.#urlUtils = deps.urlUtils;
+    }
+
+    /**
+     * @param {InstanceType<LinkRedirect>} linkRedirect 
+     * @returns {Promise<void>}
+     */
+    async save(linkRedirect) {
+        const model = await this.#LinkRedirect.add({
+            // Only store the parthname (no support for variable query strings)
+            from: this.stripSubdirectoryFromPath(linkRedirect.from.pathname),
+            to: linkRedirect.to.href
+        }, {});
+
+        linkRedirect.link_id = ObjectID.createFromHexString(model.id);
+    }
+
+    #trimLeadingSlash(url) {
+        return url.replace(/^\//, '');
+    }
+
+    fromModel(model) {
+        return new LinkRedirect({
+            id: model.id,
+            from: new URL(this.#trimLeadingSlash(model.get('from')), this.#urlUtils.urlFor('home', true)),
+            to: new URL(model.get('to'))
+        });
+    }
+
+    async getAll(options) {
+        const collection = await this.#LinkRedirect.findAll(options);
+
+        const result = [];
+
+        for (const model of collection.models) {
+            result.push(this.fromModel(model));
+        }
+
+        return result;
+    }
+
+    /**
+     * 
+     * @param {URL} url 
+     * @returns {Promise<InstanceType<LinkRedirect>|undefined>} linkRedirect 
+     */
+    async getByURL(url) {
+        // Strip subdirectory from path
+        const from = this.stripSubdirectoryFromPath(url.pathname);
+
+        const linkRedirect = await this.#LinkRedirect.findOne({
+            from
+        }, {});
+
+        if (linkRedirect) {
+            return this.fromModel(linkRedirect);
+        }
+    }
+
+    /**
+     * Convert root relative URLs to subdirectory relative URLs
+    */
+    stripSubdirectoryFromPath(path) {
+        // Bit weird, but only way to do it with the urlUtils atm
+
+        // First convert path to an absolute path
+        const absolute = this.#urlUtils.relativeToAbsolute(path);
+
+        // Then convert it to a relative path, but without subdirectory
+        return this.#urlUtils.absoluteToRelative(absolute, {withoutSubdirectory: true});
+    }
+};

package/core/server/services/link-redirection/index.js

@@ -1,4 +1,5 @@
 const urlUtils = require('../../../shared/url-utils');
+const LinkRedirectRepository = require('./LinkRedirectRepository');
 
 class LinkRedirectsServiceWrapper {
     async init() {
@@ -8,21 +9,18 @@ class LinkRedirectsServiceWrapper {
         }
 
         // Wire up all the dependencies
+        const models = require('../../models');
+
         const {LinkRedirectsService} = require('@tryghost/link-redirects');
 
-        const store = [];
+        this.linkRedirectRepository = new LinkRedirectRepository({
+            LinkRedirect: models.LinkRedirect,
+            urlUtils
+        });
+
         // Expose the service
         this.service = new LinkRedirectsService({
-            linkRedirectRepository: {
-                async save(linkRedirect) {
-                    store.push(linkRedirect);
-                },
-                async getByURL(url) {
-                    return store.find((link) => {
-                        return link.from.pathname === url.pathname;
-                    });
-                }
-            },
+            linkRedirectRepository: this.linkRedirectRepository,
             config: {
                 baseURL: new URL(urlUtils.getSiteUrl())
             }

package/core/server/services/link-tracking/LinkClickRepository.js

@@ -0,0 +1,69 @@
+const {LinkClick} = require('@tryghost/link-tracking');
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkClickRepository {
+    /** @type {Object} */
+    #MemberLinkClickEventModel;
+
+    /** @type {Object} */
+    #MemberLinkClickEvent;
+
+    /** @type {object} */
+    #Member;
+
+    /** @type {object} */
+    #DomainEvents;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MemberLinkClickEventModel Bookshelf Model
+     * @param {object} deps.Member Bookshelf Model
+     * @param {object} deps.MemberLinkClickEvent Event
+     * @param {object} deps.DomainEvents
+     */
+    constructor(deps) {
+        this.#MemberLinkClickEventModel = deps.MemberLinkClickEventModel;
+        this.#Member = deps.Member;
+        this.#MemberLinkClickEvent = deps.MemberLinkClickEvent;
+        this.#DomainEvents = deps.DomainEvents;
+    }
+
+    async getAll(options) {
+        const collection = await this.#MemberLinkClickEventModel.findAll(options);
+
+        const result = [];
+
+        for (const model of collection.models) {
+            const member = await this.#Member.findOne({id: model.get('member_id')});
+            result.push(new LinkClick({
+                link_id: model.get('link_id'),
+                member_uuid: member.get('uuid')
+            }));
+        }
+
+        return result;
+    }
+
+    /**
+     * @param {LinkClick} linkClick
+     * @returns {Promise<void>}
+     */
+    async save(linkClick) {
+        // Convert uuid to id
+        const member = await this.#Member.findOne({uuid: linkClick.member_uuid});
+        if (!member) {
+            return;
+        }
+
+        const model = await this.#MemberLinkClickEventModel.add({
+            // Only store the parthname (no support for variable query strings)
+            link_id: linkClick.link_id.toHexString(), 
+            member_id: member.id
+        }, {});
+
+        linkClick.event_id = ObjectID.createFromHexString(model.id);
+
+        // Dispatch event
+        this.#DomainEvents.dispatch(this.#MemberLinkClickEvent.create({memberId: member.id, memberLastSeenAt: member.get('last_seen_at'), linkId: linkClick.link_id.toHexString()}, new Date()));
+    }
+};