ghost 5.115.1 → 5.116.0

package/core/server/services/link-redirection/LinkRedirectsService.js

@@ -0,0 +1,123 @@
+const crypto = require('crypto');
+const DomainEvents = require('@tryghost/domain-events');
+const RedirectEvent = require('./RedirectEvent');
+const LinkRedirect = require('./LinkRedirect');
+
+/**
+ * @typedef {object} ILinkRedirectRepository
+ * @prop {(url: URL) => Promise<LinkRedirect|undefined>} getByURL
+ * @prop {({filter: string}) => Promise<LinkRedirect[]>} getAll
+ * @prop {({filter: string}) => Promise<String[]>} getFilteredIds
+ * @prop {(linkRedirect: LinkRedirect) => Promise<void>} save
+ */
+
+class LinkRedirectsService {
+    /** @type ILinkRedirectRepository */
+    #linkRedirectRepository;
+    /** @type URL */
+    #baseURL;
+
+    /** @type String */
+    #redirectURLPrefix = 'r/';
+
+    /**
+     * @param {object} deps
+     * @param {ILinkRedirectRepository} deps.linkRedirectRepository
+     * @param {object} deps.config
+     * @param {URL} deps.config.baseURL
+     */
+    constructor(deps) {
+        this.#linkRedirectRepository = deps.linkRedirectRepository;
+        if (!deps.config.baseURL.pathname.endsWith('/')) {
+            this.#baseURL = new URL(deps.config.baseURL);
+            this.#baseURL.pathname += '/';
+        } else {
+            this.#baseURL = deps.config.baseURL;
+        }
+        this.handleRequest = this.handleRequest.bind(this);
+    }
+
+    /**
+     * Get a unique URL with slug for creating unique redirects
+     *
+     * @returns {Promise<URL>}
+     */
+    async getSlugUrl() {
+        let url;
+        while (!url || await this.#linkRedirectRepository.getByURL(url)) {
+            const slug = crypto.randomBytes(4).toString('hex');
+            url = new URL(`${this.#redirectURLPrefix}${slug}`, this.#baseURL);
+        }
+        return url;
+    }
+
+    /**
+     * @param {Object} options
+     *
+     * @returns {Promise<String[]>}
+     */
+    async getFilteredIds(options) {
+        return await this.#linkRedirectRepository.getFilteredIds(options);
+    }
+
+    /**
+     * @param {URL} from
+     * @param {URL} to
+     *
+     * @returns {Promise<LinkRedirect>}
+     */
+    async addRedirect(from, to) {
+        const link = new LinkRedirect({
+            from,
+            to
+        });
+
+        await this.#linkRedirectRepository.save(link);
+
+        return link;
+    }
+
+    /**
+     * @param {import('express').Request} req
+     * @param {import('express').Response} res
+     * @param {import('express').NextFunction} next
+     *
+     * @returns {Promise<void>}
+     */
+    async handleRequest(req, res, next) {
+        try {
+            // skip handling if original url doesn't match the prefix
+            const fullURLWithRedirectPrefix = `${this.#baseURL.pathname}${this.#redirectURLPrefix}`;
+            // @NOTE: below is equivalent to doing:
+            //          router.get('/r/'), (req, res) ...
+            //        To make it cleaner we should rework it to:
+            //          linkRedirects.service.handleRequest(router);
+            //        and mount routes on top like for example sitemapHandler does
+            //        Cleanup issue: https://github.com/TryGhost/Toolbox/issues/516
+            if (!req.originalUrl.startsWith(fullURLWithRedirectPrefix)) {
+                return next();
+            }
+
+            const url = new URL(req.originalUrl, this.#baseURL);
+            const link = await this.#linkRedirectRepository.getByURL(url);
+
+            if (!link) {
+                return next();
+            }
+
+            const event = RedirectEvent.create({
+                url,
+                link
+            });
+
+            DomainEvents.dispatch(event);
+
+            res.setHeader('X-Robots-Tag', 'noindex, nofollow');
+            return res.redirect(link.to.href);
+        } catch (e) {
+            return next(e);
+        }
+    }
+}
+
+module.exports = LinkRedirectsService;

package/core/server/services/link-redirection/README.md

@@ -0,0 +1,151 @@
+# Link Redirects
+
+
+## Usage
+
+
+## Develop
+
+This is a monorepo package.
+
+Follow the instructions for the top-level repo.
+1. `git clone` this repo & `cd` into it as usual
+2. Run `yarn` to install top-level dependencies.
+
+
+
+## Test
+
+- `yarn lint` run just eslint
+- `yarn test` run lint and tests
+
+## Overview of how Ghost handles LinkRedirects
+### Summary
+When a publisher sends an email newsletter with email click analytics enabled, Ghost will replace all the links in the email's content with a link of the form `https://{site_url}/r/{redirect hash}?m={member UUID}`. When a member clicks on a link in their email, Ghost receives the request, redirects the user to the original link's URL, then updates some analytics data in the database.
+
+### The details
+The following deep-dive covers the link redirect flow from when the member clicks on a link in an email.
+
+First, we lookup the redirect by the `/r/{hash}` value in the URL:
+```
+select `redirects`.* from `redirects` where `redirects`.`from` = ? limit ? undefined
+```
+
+If the redirect exists, the `LinkRedirectsService` emits a `RedirectEvent`, and then responds to the HTTP request with a 302. 
+
+The `LinkClickTrackingService` subscribes to the `RedirectEvent` and kicks off the analytics inserts/updates. First we grab the `uuid` from the `?m={uuid}` parameter and lookup the member by `uuid`:
+```
+select `members`.* from `members` where `members`.`uuid` = ? limit ? undefined
+```
+
+Then we insert a row into the `members_click_events` table to record the click:
+```
+insert into `members_click_events` (`created_at`, `id`, `member_id`, `redirect_id`) values (?, ?, ?, ?) undefined
+```
+
+Then we query for the row we just inserted:
+```
+select `members_click_events`.* from `members_click_events` where `members_click_events`.`id` = ? limit ? undefined
+```
+
+At this point, we emit a `MemberLinkClickEvent` with the member ID and `lastSeenAt` timestamp. 
+
+The `LastSeenAtUpdater` subscribes to the `MemberLinkClickEvent`. First, it checks if the `lastSeenAt` value has already been updated in the current day. 
+
+If it has, we stop here. 
+
+If it hasn't, we continue to update the member. First, we select the member by ID:
+```
+select `members`.* from `members` where `members`.`id` = ? limit ? undefined
+```
+
+Then we start a transaction and get a lock on the member for updating:
+```
+BEGIN; trx34
+select `members`.* from `members` where `members`.`id` = ? limit ? for update trx34
+```
+
+Since we're editing the member, we will eventually need to emit a `member.edited` webhook with the standard includes (labels and newsletters) so we also query them here:
+```
+select `labels`.*, `members_labels`.`member_id` as `_pivot_member_id`, `members_labels`.`label_id` as `_pivot_label_id`, `members_labels`.`sort_order` as `_pivot_sort_order` from `labels` inner join `members_labels` on `members_labels`.`label_id` = `labels`.`id` where `members_labels`.`member_id` in (?) order by `sort_order` ASC for update trx34
+```
+
+Then we query the member's newsletters:
+```
+select `newsletters`.*, `members_newsletters`.`member_id` as `_pivot_member_id`, `members_newsletters`.`newsletter_id` as `_pivot_newsletter_id` from `newsletters` inner join `members_newsletters` on `members_newsletters`.`newsletter_id` = `newsletters`.`id` where `members_newsletters`.`member_id` in (?) order by `newsletters`.`sort_order` ASC for update trx34
+```
+
+Then we update the member:
+```
+update `members` set `uuid` = ?, `transient_id` = ?, `email` = ?, `status` = ?, `name` = ?, `expertise` = ?, `note` = ?, `geolocation` = ?, `enable_comment_notifications` = ?, `email_count` = ?, `email_opened_count` = ?, `email_open_rate` = ?, `email_disabled` = ?, `last_seen_at` = ?, `last_commented_at` = ?, `created_at` = ?, `created_by` = ?, `updated_at` = ?, `updated_by` = ? where `id` = ? trx34
+```
+
+Then we select the member by ID again to get the freshly updated values from the DB:
+```
+select `members`.* from `members` where `members`.`id` = ? limit ? trx34
+```
+
+Then we commit the transaction:
+```
+COMMIT; trx34
+```
+
+Finally, we query for any member.edited webhooks and fire the `member.edited` event:
+```
+select `webhooks`.* from `webhooks` where `event` = ? trx34
+```
+
+
+### Sequence Diagram
+```mermaid
+sequenceDiagram
+    actor Member
+    participant Ghost
+    participant Ghost Async
+    participant DB
+    rect rgba(0,100,0, 0.1)
+    Member ->>Ghost: Clicks link in email
+    Ghost ->> DB: Query: lookup redirect
+    DB ->> Ghost: Redirect record
+    Note right of DB: Serve the redirect
+    Ghost -->> Ghost Async: Emit RedirectEvent
+    Ghost ->> Member: 302 Redirect
+    end
+    rect rgba(100,0,0,0.1)
+    Ghost Async ->> DB: Lookup Member by `uuid` from URL param
+    DB ->> Ghost Async: `member` record
+    Ghost Async ->> DB: Insert `member_click_event`
+    Note right of DB: Insert click event
+    DB ->> Ghost Async: 👌
+    Ghost Async ->> DB: Select `member_click_event`
+    DB ->> Ghost Async: `member_click_event` record
+    end
+    rect rgba(0,0,100, 0.1)
+    Ghost Async ->> DB: Select `member` by id
+    DB ->> Ghost Async: `member` record
+    Ghost Async ->> DB: Begin transaction 
+    activate DB
+    DB ->> Ghost Async: 👌
+    Ghost Async ->> DB: Select `member` for update
+    DB ->> Ghost Async: `member` record
+    Ghost Async ->> DB: Select member labels for update
+    Note right of DB: Update member `lastSeenAt`
+    DB ->> Ghost Async: Member's labels
+    Ghost Async ->> DB: Select member newsletters for update
+    DB ->> Ghost Async: Member's newsletters
+    Ghost Async ->> DB: Update member's `lastSeenAt` timestamp
+    DB ->> Ghost Async: 👌
+    Ghost Async ->> DB: Select `member` by ID
+    DB ->> Ghost Async: `member` record
+    Ghost Async ->> DB: Commit transaction
+    DB ->> Ghost Async: 👌
+    deactivate DB
+    end
+    rect rgba(100,100,0,0.1)
+    Ghost Async ->> DB: Select `webhooks`
+    Note right of DB: Send `member.edited` webhook 
+    DB ->> Ghost Async: `webhook` records
+    create participant Webhook Recipient
+    Ghost Async ->> Webhook Recipient: `member.edited` webhook
+    end
+```

package/core/server/services/link-redirection/RedirectEvent.js

@@ -0,0 +1,24 @@
+/**
+ * @typedef {object} RedirectEventData
+ * @prop {URL} url
+ * @prop {import('./LinkRedirect')} link
+ */
+
+module.exports = class RedirectEvent {
+    /**
+     * @param {RedirectEventData} data
+     * @param {Date} timestamp
+     */
+    constructor(data, timestamp) {
+        this.data = data;
+        this.timestamp = timestamp;
+    }
+
+    /**
+     * @param {RedirectEventData} data
+     * @param {Date} [timestamp]
+     */
+    static create(data, timestamp) {
+        return new RedirectEvent(data, timestamp ?? new Date);
+    }
+};

package/core/server/services/link-redirection/index.js

@@ -14,7 +14,7 @@ class LinkRedirectsServiceWrapper {
         // Wire up all the dependencies
         const models = require('../../models');
 
-        const {LinkRedirectsService} = require('@tryghost/link-redirects');
+        const LinkRedirectsService = require('./LinkRedirectsService');
 
         this.linkRedirectRepository = new LinkRedirectRepository({
             LinkRedirect: models.Redirect,

package/core/server/services/link-tracking/LinkClickTrackingService.js

@@ -1,4 +1,4 @@
-const {RedirectEvent} = require('@tryghost/link-redirects');
+const RedirectEvent = require('../link-redirection/RedirectEvent');
 const LinkClick = require('./ClickEvent');
 const PostLink = require('./PostLink');
 const ObjectID = require('bson-objectid').default;

package/core/server/services/mail/index.js

@@ -1,7 +1,7 @@
 const path = require('path');
 const urlUtils = require('../../../shared/url-utils');
 const settingsCache = require('../../../shared/settings-cache');
-const EmailContentGenerator = require('@tryghost/email-content-generator');
+const EmailContentGenerator = require('../lib/EmailContentGenerator');
 
 const emailContentGenerator = new EmailContentGenerator({
     getSiteUrl: () => urlUtils.urlFor('home', true),

package/core/server/services/mail-events/InMemoryMailEventRepository.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryMailEventRepository = void 0;
-const in_memory_repository_1 = require("@tryghost/in-memory-repository");
-class InMemoryMailEventRepository extends in_memory_repository_1.InMemoryRepository {
+const InMemoryRepository_1 = require("../lib/InMemoryRepository");
+class InMemoryMailEventRepository extends InMemoryRepository_1.InMemoryRepository {
     toPrimitive() {
         return {};
     }

package/core/server/services/mail-events/InMemoryMailEventRepository.ts

@@ -1,4 +1,4 @@
-import {InMemoryRepository} from '@tryghost/in-memory-repository';
+import {InMemoryRepository} from '../lib/InMemoryRepository';
 import {MailEvent} from './MailEvent';
 
 export class InMemoryMailEventRepository extends InMemoryRepository<string, MailEvent> {

package/core/server/services/members-events/LastSeenAtUpdater.js

@@ -1,7 +1,7 @@
 const {MemberPageViewEvent, MemberCommentEvent, MemberLinkClickEvent} = require('@tryghost/member-events');
 const moment = require('moment-timezone');
 const {IncorrectUsageError} = require('@tryghost/errors');
-const {EmailOpenedEvent} = require('@tryghost/email-events');
+const {EmailOpenedEvent} = require('@tryghost/email-service');
 const logging = require('@tryghost/logging');
 const LastSeenAtCache = require('./LastSeenAtCache');
 

package/core/server/services/offers/service.js

@@ -1,6 +1,6 @@
-const DynamicRedirectManager = require('@tryghost/express-dynamic-redirects');
 const OffersModule = require('@tryghost/members-offers');
 
+const DynamicRedirectManager = require('../lib/DynamicRedirectManager');
 const config = require('../../../shared/config');
 const urlUtils = require('../../../shared/url-utils');
 const models = require('../../models');

package/core/server/services/recommendations/RecommendationServiceWrapper.js

@@ -4,37 +4,37 @@ const logging = require('@tryghost/logging');
 
 class RecommendationServiceWrapper {
     /**
-     * @type {import('@tryghost/recommendations').RecommendationRepository}
+     * @type {import('./service').RecommendationRepository}
      */
     repository;
 
     /**
-     * @type {import('@tryghost/recommendations').BookshelfClickEventRepository}
+     * @type {import('./service').BookshelfClickEventRepository}
      */
     clickEventRepository;
 
     /**
-     * @type {import('@tryghost/recommendations').BookshelfSubscribeEventRepository}
+     * @type {import('./service').BookshelfSubscribeEventRepository}
      */
     subscribeEventRepository;
 
     /**
-     * @type {import('@tryghost/recommendations').RecommendationController}
+     * @type {import('./service').RecommendationController}
      */
     controller;
 
     /**
-     * @type {import('@tryghost/recommendations').RecommendationService}
+     * @type {import('./service').RecommendationService}
      */
     service;
 
     /**
-     * @type {import('@tryghost/recommendations').IncomingRecommendationController}
+     * @type {import('./service').IncomingRecommendationController}
      */
     incomingRecommendationController;
 
     /**
-     * @type {import('@tryghost/recommendations').IncomingRecommendationService}
+     * @type {import('./service').IncomingRecommendationService}
      */
     incomingRecommendationService;
 
@@ -65,7 +65,7 @@ class RecommendationServiceWrapper {
             IncomingRecommendationService,
             IncomingRecommendationEmailRenderer,
             RecommendationMetadataService
-        } = require('@tryghost/recommendations');
+        } = require('./service');
 
         const mentions = require('../mentions');
 

package/core/server/services/recommendations/service/BookshelfClickEventRepository.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BookshelfClickEventRepository = void 0;
+const BookshelfRepository_1 = require("./BookshelfRepository");
+const logging_1 = __importDefault(require("@tryghost/logging"));
+const ClickEvent_1 = require("./ClickEvent");
+class BookshelfClickEventRepository extends BookshelfRepository_1.BookshelfRepository {
+    sentry;
+    constructor(Model, deps = {}) {
+        super(Model);
+        this.sentry = deps.sentry;
+    }
+    toPrimitive(entity) {
+        return {
+            id: entity.id,
+            recommendation_id: entity.recommendationId,
+            member_id: entity.memberId,
+            created_at: entity.createdAt
+        };
+    }
+    modelToEntity(model) {
+        try {
+            return ClickEvent_1.ClickEvent.create({
+                id: model.id,
+                recommendationId: model.get('recommendation_id'),
+                memberId: model.get('member_id'),
+                createdAt: model.get('created_at')
+            });
+        }
+        catch (err) {
+            logging_1.default.error(err);
+            this.sentry?.captureException(err);
+            return null;
+        }
+    }
+    getFieldToColumnMap() {
+        return {
+            id: 'id',
+            recommendationId: 'recommendation_id',
+            memberId: 'member_id',
+            createdAt: 'created_at'
+        };
+    }
+}
+exports.BookshelfClickEventRepository = BookshelfClickEventRepository;

package/core/server/services/recommendations/service/BookshelfClickEventRepository.ts

@@ -0,0 +1,49 @@
+import {BookshelfRepository, ModelClass, ModelInstance} from './BookshelfRepository';
+import logger from '@tryghost/logging';
+import {ClickEvent} from './ClickEvent';
+
+type Sentry = {
+    captureException(err: unknown): void;
+}
+
+export class BookshelfClickEventRepository extends BookshelfRepository<string, ClickEvent> {
+    sentry?: Sentry;
+
+    constructor(Model: ModelClass<string>, deps: {sentry?: Sentry} = {}) {
+        super(Model);
+        this.sentry = deps.sentry;
+    }
+
+    toPrimitive(entity: ClickEvent): object {
+        return {
+            id: entity.id,
+            recommendation_id: entity.recommendationId,
+            member_id: entity.memberId,
+            created_at: entity.createdAt
+        };
+    }
+
+    modelToEntity(model: ModelInstance<string>): ClickEvent | null {
+        try {
+            return ClickEvent.create({
+                id: model.id,
+                recommendationId: model.get('recommendation_id') as string,
+                memberId: model.get('member_id') as string | null,
+                createdAt: model.get('created_at') as Date
+            });
+        } catch (err) {
+            logger.error(err);
+            this.sentry?.captureException(err);
+            return null;
+        }
+    }
+
+    getFieldToColumnMap() {
+        return {
+            id: 'id',
+            recommendationId: 'recommendation_id',
+            memberId: 'member_id',
+            createdAt: 'created_at'
+        } as Record<keyof ClickEvent, string>;
+    }
+}

package/core/server/services/recommendations/service/BookshelfRecommendationRepository.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BookshelfRecommendationRepository = void 0;
+const BookshelfRepository_1 = require("./BookshelfRepository");
+const logging_1 = __importDefault(require("@tryghost/logging"));
+const Recommendation_1 = require("./Recommendation");
+class BookshelfRecommendationRepository extends BookshelfRepository_1.BookshelfRepository {
+    sentry;
+    constructor(Model, deps = {}) {
+        super(Model);
+        this.sentry = deps.sentry;
+    }
+    applyCustomQuery(query, options) {
+        query.select('recommendations.*');
+        if (options.include?.includes('clickCount') || options.order?.find(o => o.field === 'clickCount')) {
+            query.select((knex) => {
+                knex.count('*').from('recommendation_click_events').where('recommendation_click_events.recommendation_id', knex.client.raw('recommendations.id')).as('count__clicks');
+            });
+        }
+        if (options.include?.includes('subscriberCount') || options.order?.find(o => o.field === 'subscriberCount')) {
+            query.select((knex) => {
+                knex.count('*').from('recommendation_subscribe_events').where('recommendation_subscribe_events.recommendation_id', knex.client.raw('recommendations.id')).as('count__subscribers');
+            });
+        }
+    }
+    toPrimitive(entity) {
+        return {
+            id: entity.id,
+            title: entity.title,
+            description: entity.description,
+            excerpt: entity.excerpt,
+            featured_image: entity.featuredImage?.toString(),
+            favicon: entity.favicon?.toString(),
+            url: entity.url.toString(),
+            one_click_subscribe: entity.oneClickSubscribe,
+            created_at: entity.createdAt,
+            updated_at: entity.updatedAt
+            // Count relations are not saveable: so don't set them here
+        };
+    }
+    modelToEntity(model) {
+        try {
+            return Recommendation_1.Recommendation.create({
+                id: model.id,
+                title: model.get('title'),
+                description: model.get('description'),
+                excerpt: model.get('excerpt'),
+                featuredImage: model.get('featured_image'),
+                favicon: model.get('favicon'),
+                url: model.get('url'),
+                oneClickSubscribe: model.get('one_click_subscribe'),
+                createdAt: model.get('created_at'),
+                updatedAt: model.get('updated_at'),
+                clickCount: (model.get('count__clicks') ?? undefined),
+                subscriberCount: (model.get('count__subscribers') ?? undefined)
+            });
+        }
+        catch (err) {
+            logging_1.default.error(err);
+            this.sentry?.captureException(err);
+            return null;
+        }
+    }
+    getFieldToColumnMap() {
+        return {
+            id: 'id',
+            title: 'title',
+            description: 'description',
+            excerpt: 'excerpt',
+            featuredImage: 'featured_image',
+            favicon: 'favicon',
+            url: 'url',
+            oneClickSubscribe: 'one_click_subscribe',
+            createdAt: 'created_at',
+            updatedAt: 'updated_at',
+            clickCount: 'count__clicks',
+            subscriberCount: 'count__subscribers'
+        };
+    }
+    async getByUrl(url) {
+        const urlFilter = `url:~'${url.host.replace('www.', '')}${url.pathname.replace(/\/$/, '')}'`;
+        const recommendations = await this.getAll({ filter: urlFilter });
+        if (!recommendations || recommendations.length === 0) {
+            return null;
+        }
+        //  Find URL based on the hostname and pathname.
+        //  Query params, hash fragements, protocol and www are ignored.
+        const existing = recommendations.find((r) => {
+            return r.url.hostname.replace('www.', '') === url.hostname.replace('www.', '') &&
+                r.url.pathname.replace(/\/$/, '') === url.pathname.replace(/\/$/, '');
+        }) || null;
+        return existing;
+    }
+}
+exports.BookshelfRecommendationRepository = BookshelfRecommendationRepository;